{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import time\n",
    "from sys import getsizeof\n",
    "\n",
    "# preprocessing\n",
    "from config import DATA_CSV, PIXEL_ARRAY, DTYPE, Y_LABEL\n",
    "from config import FEATURES, OUTPUT_FOLDER, FORMAT\n",
    "from preprocessing import to_arr\n",
    "from imblearn.under_sampling import RandomUnderSampler\n",
    "from imblearn.over_sampling import ADASYN, SMOTE, RandomOverSampler\n",
    "\n",
    "# image processing\n",
    "# from model import run_model, run_desc_model\n",
    "from image_processing import sift_ext, surf_ext, orb_ext\n",
    "from image_processing import sift_des, surf_des, orb_des\n",
    "from image_processing import generate_bag\n",
    "# from image_processing import generate_histo\n",
    "\n",
    "# pipeline\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from config import TRANSFORMATION_LIST, EXTRACTION_LIST\n",
    "from sklearn.preprocessing import FunctionTransformer\n",
    "\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import silhouette_score\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from scipy.stats import randint as sp_randint\n",
    "from sklearn.metrics import accuracy_score, log_loss\n",
    "\n",
    "# model\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC, LinearSVC, NuSVC\n",
    "from sklearn.linear_model import LogisticRegression, SGDClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "RANDOM_STATE = 0\n",
    "# import warnings\n",
    "# warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = np.load(PIXEL_ARRAY, allow_pickle=True)\n",
    "data = pd.DataFrame({'Pixel': data})\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([1506,  625]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.load(Y_LABEL)\n",
    "np.unique(y, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.load(OUTPUT_FOLDER + 'hl.npy', allow_pickle=True)\n",
    "# X_arr = to_arr(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(258, 258)\n",
      "(66564,)\n"
     ]
    }
   ],
   "source": [
    "print(X[0].shape) #wavelet\n",
    "print(X_arr[0].shape) #must"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1999, 128)\n",
      "(262144,)\n"
     ]
    }
   ],
   "source": [
    "print(X[0].shape) #sift\n",
    "print(X_arr[0].shape) #cannot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(512, 512)\n",
      "(262144,)\n"
     ]
    }
   ],
   "source": [
    "print(X[0].shape) #fft\n",
    "print(X_arr[0].shape) #must"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random over sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ros = RandomOverSampler(random_state=0)\n",
    "ros.fit(X_arr, y)\n",
    "X_res, y_res = ros.fit_resample(X_arr, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([1506, 1506]))"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(y_res, return_counts=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ADASYN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = ADASYN(random_state=0)\n",
    "smote.fit(X_arr, y)\n",
    "X_res, y_res = smote.fit_resample(X_arr, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformation_transformer = Pipeline(steps = [\n",
    "    ('to_arr', FunctionTransformer(to_arr))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_trans = transformation_transformer.transform(X)\n",
    "# X = transformation_transformer.fit(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array_equal(X_arr, X_trans)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTE(random_state=RANDOM_STATE)\n",
    "smote.fit(X_trans, y)\n",
    "X_res, y_res = smote.fit_resample(X_trans, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTE(random_state=RANDOM_STATE)\n",
    "smote.fit(X_arr, y)\n",
    "X_res, y_res = smote.fit_resample(X_arr, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "img_col = np.load(OUTPUT_FOLDER + 'sift.npy', allow_pickle=True)\n",
    "\n",
    "# col = np.vstack(np.uint32(i) for i in img_col)\n",
    "# print(np.array_equal(bag, col))\n",
    "bag = generate_bag(img_col)\n",
    "\n",
    "print(getsizeof(bag))\n",
    "print(col.nbytes)\n",
    "\n",
    "print(len(img_col))\n",
    "print(np.shape(bag))\n",
    "print(col.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = 2 * 10\n",
    "batch_size = 45"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mbk = MiniBatchKMeans(n_clusters=k, batch_size=batch_size, n_init=10, max_no_improvement=10, verbose=0)\n",
    "t0 = time.time()\n",
    "mbk.fit(bag)\n",
    "t_mini_batch = time.time() - t0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111.65457677841187"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_mini_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "238.77364921569824"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t_mini_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "histo_list = []\n",
    "for descriptors in img_col:\n",
    "#     print(descriptors)\n",
    "#     print(\"+++++++++\")\n",
    "    histo = np.zeros(k)\n",
    "    nkp = np.size(descriptors)\n",
    "\n",
    "    for d in descriptors:\n",
    "        idx = mbk.predict([d])\n",
    "        histo[idx] += 1 / nkp  # Because we need normalized histograms, I prefere to add 1/nkp directly\n",
    "\n",
    "    histo_list.append(histo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([0.0003791 , 0.00035174, 0.00089498, 0.00044163, 0.00029702,\n",
       "        0.00033611, 0.00038691, 0.00028921, 0.00023058, 0.00026185,\n",
       "        0.00019541, 0.00021495, 0.0009028 , 0.00020713, 0.00049243,\n",
       "        0.00055887, 0.00025403, 0.00065658, 0.00024231, 0.00021886]),\n",
       " array([0.00048114, 0.00031499, 0.0005296 , 0.0004396 , 0.00037038,\n",
       "        0.00031845, 0.00049499, 0.00034961, 0.00029768, 0.00034268,\n",
       "        0.00030115, 0.00036691, 0.00054691, 0.00025269, 0.00053306,\n",
       "        0.0006196 , 0.00027345, 0.00041884, 0.00024922, 0.00031153]),\n",
       " array([0.00046754, 0.00031295, 0.00031672, 0.00050525, 0.00046754,\n",
       "        0.00040722, 0.00050902, 0.00032049, 0.00020738, 0.00042984,\n",
       "        0.00035066, 0.00037328, 0.00038082, 0.00024885, 0.00047885,\n",
       "        0.00066361, 0.00028279, 0.00040344, 0.00025639, 0.00042984]),\n",
       " array([0.0003249 , 0.00033237, 0.00041826, 0.00037718, 0.0004556 ,\n",
       "        0.00029129, 0.00051909, 0.00048921, 0.00027635, 0.00038838,\n",
       "        0.00030623, 0.00035477, 0.00042573, 0.00026141, 0.00047801,\n",
       "        0.00068341, 0.0003249 , 0.00046681, 0.00027262, 0.00036598]),\n",
       " array([0.00047695, 0.0003587 , 0.00030351, 0.00051637, 0.00037841,\n",
       "        0.00038629, 0.00058338, 0.00039417, 0.00020497, 0.00042176,\n",
       "        0.00030745, 0.00037841, 0.00027986, 0.00020103, 0.00059126,\n",
       "        0.00075681, 0.00030351, 0.00041782, 0.00026015, 0.00029169])]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "histo_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing import pca_curve, pca_ft"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3deXwddb3/8den2Zt0SZvuO7SFtlBKqaWsIhQpilRBryyCKIoLoKhXL/68F716r9cFFb24AbIpiiyKvQqyySpbF0pXutA1pW2SNnua/fP7YybtaWmSaZvJOcl5Px+P88jMnDlzPmeSfD9nvt/5fr/m7oiISPrqk+wAREQkuZQIRETSnBKBiEiaUyIQEUlzSgQiImkuM9kBHKqioiIfP358ssMQEelRFi9eXObuQw72XI9LBOPHj2fRokXJDkNEpEcxs83tPaeqIRGRNKdEICKS5pQIRETSnBKBiEiaUyIQEUlzsSUCM7vTzErMbEU7z5uZ/czM1pvZMjObGVcsIiLSvjivCO4G5nXw/PnApPBxDfDLGGMREZF2xNaPwN2fN7PxHewyH7jXg3GwXzGzgWY2wt23xxWTiEiyuDsNza3UN7Wwp6mFPY3Bz/qmFuqbWmlsbqWhuZXGlmA5eLTst37OlGGcMGZgl8eWzA5lo4CtCevF4bZ3JAIzu4bgqoGxY8d2S3Aikp7cnT1NLdQ0NFPb0EJtQ3O43HzQbbWNQWHeVrC3Fe6JBX1duHyk078M7Z/b6xJBZO5+G3AbwKxZszSTjoi0q76phao9TVTVN1G5p3nvctWeJir3NFFV35yw3ERNWLAnFu6tEUuZvKwM8nMyyMvOIC8rfGRnMDg/m7zCDHKz9t+em7CclxWuZ2eQm9mH7PCRk9mH7IyMvevZmX3IzuhDVoZhZrGcs2Qmgm3AmIT10eE2ERFaW53KPU3srmukoq6R3bVNlNc2sruukfLaRsrbtoXPV+5ppqq+icbm1g6Pm5vVhwF5WfTPzaJfbiYD8rIYNTCX/OxM8nMyKchp+5lBfk772/KzM8noE0/B3N2SmQgWANeZ2f3AyUCl2gdEei93p7axhbLqBspqGiht+1nTSFlNA2XVDewOC/jyuiYq6hrb/WaendmHQX2zKczPZlB+FscO70//vCz652XSPzcrKOjzwp+5mXuX++VmkpOZ0b0fvAeILRGY2R+As4AiMysGvglkAbj7r4BHgfcB64E64BNxxSIi8WlpdXbVNLCjqp7tlfWUVNVTWp1QwNfsK/jrm975bd0MCvtmU1SQzeD8HI4d3p/C/CwK+2ZT2DebQflhgd83e+/2vtkZsVWTpKM47xq6tJPnHbg2rvcXkSNX39RCSVVbIb+HnWFhv7Oqnh2VwaOkuoHmA766m8Hg/GyKCnIoKshh3Ni+wXK/HIaEP4sKshlSkMOg/GwyM9S3NZl6RGOxiMSjur6JbRV7KN69h+LyOorL9wSPijrerqhnd23jO17TNzuD4QNyGTEglzlHD2bEgFyG989l+IA8hvfPZVh/Fe49jRKBSC/W2NzK1vI6Nu+qZcuu/Qv64vI9VNQ17bd/blYfRhf2ZXRhHsePGhgU8mFBP2JALsMG5NIvJ1PVMr2MEoFID9fU0kpx+R42ldWyaVctm8pq2birjk1ltWyr2ENLQrVNXlYGowvzGFWYx4wxA/cW+m0/B+dnq5BPQ0oEIj1E5Z4m1pdUs25nDWt31rChrIZNZbUUl+/Zr46+X04m44vymT56APNnjGT84HzGF+UzbnBfFfRyUEoEIimmsq6JtWGBvy7h586qhr375Gb14aiiAqaNHMD7p49g/OB8JhQFBb4KezlUSgQiSdLQ3MK6nTWs2l7F6u1VrN1ZzdqdNZRW7yvw87IymDSsgNMmFjFpaD8mDytg0tB+jC7Mo08v6cwkyadEININSqsbWB0W+MGjmvWlNXvr73Oz+jB5WD/OnDQkKOzDAn/UQBX4Ej8lApEuVl7byBvFFbyxtZJlxRUs21a537f84f1zmTqyP3OnDmXKiP5MGdGf8YPze81wBdLzKBGIHIHahmZWbKtkWXFlUPgXV7B19x4g6FR19JACzphYxNSR/ZkaFvqF+dlJjlpkf0oEIhG1tDprdlSzZEs5b2ytYFlxJetKqveOhzNqYB7TRw/g8pPHMX30AI4fNYB+uVnJDVokAiUCkXZU1zexdGsFizeXs3hzOa9vqaCmoRmAwr5ZTB89kPOOG84JowcwffRAhvTLSXLEIodHiUAkVFHXyCsbdvPKhl28unE3a3ZU0epBFc8xw/oxf8ZIZo0vZObYQsYO6qtbNKXXUCKQtFVd38RrG3fz8lu7eHnDLlZtr8I9uIPnpHGFXH/2JE4aV8iMsQPpryoe6cWUCCRt1De18NrG3bwUFvzLiyto9WBs+5ljB3LDOZM55ejBnDBmgMasl7SiRCC9lruzoayW59aU8tzaUl7ZsIuG5lYy+xgzxgzk2vdM5JSjBjNzXCG5WSr4JX0pEUiv0tLqLNlSzuMrdvDEqp1s2V0HwFFD8rns5LGcOXkIJ08YRN9s/emLtNF/g/R4Dc0tvLR+F4+v3MFTq3dSVtNIdkYfTps4mE+feRRnTR7CmEF9kx2mSMpSIpAeqbq+iWfWlPL4yh08+2YJtY0tFORkctYxQzhv2nDOOmaI7uEXiShSIjCz04FJ7n6XmQ0BCtx9Y7yhieyvrrGZp1aXsGDp2zy/tpTGllaKCrK5cMZI3jttOKcePViNvCKHodNEYGbfBGYBxwB3EUxA/zvgtHhDEwlm2HpubSkL3nibp1btZE9TC8P753LlKeOYd9xwThxbqDF6RI5QlCuCDwEnAksA3P1tM+sXa1SS1tydVzfu5s9LtvHYiu1U1TdT2DeLi2aO4sITRvKu8YM0IqdIF4qSCBrd3c3MAcwsP+aYJE1tr9zDw4uLeXBxMZt31ZGfncF504bzgRkjOX1iEVmaDF0kFlESwQNm9mtgoJl9GvgkcHu8YUm6aGhu4enVJfxx4VZeWFdKq8MpRw3mhrmTmDdtBHnZqvMXiVunicDdbzazc4EqgnaCm9z9ydgjk15t9fYqHli0lUde30Z5XRMjBuRy7Xsm8uGTRjNusC46RbpTlMbiCcALbYW/meWZ2Xh33xR3cNK71De1sOCNt/ndK5tZVlxJdkYfzp02jH+ZNYbTJxap0VckSaJUDT0InJqw3hJue1csEUmv83bFHn73ymb+8NoWyuuamDysgG9+YCofnDFKk7SIpIAoiSDT3RvbVty90cz03ysdarvz556XNvHEqp24O3OnDOOq08ZzylGDNYSzSAqJkghKzexCd18AYGbzgbJ4w5KeqqG5hQVL3+Y3L27kzR3VDMjL4lNnTOBjJ4/TMA8iKSpKIvgscJ+Z3QoYsBW4MtaopMep3NPEfa9u5u5/bqKkuoFjhvXjexcdz/wZo3Tnj0iKi3LX0FvAHDMrCNdrYo9KeowdlfXc9vwG/rhwC7WNLZw+sYgffuQEzpxUpOofkR4iyl1DOcDFwHggs+2f292/HWtkktKKy+v45bNv8eCiYlrc+cD0EXz6zKOYNnJAskMTkUMUpWroL0AlsBhoiDccSXVbdtXx82fW8/CSYszgwyeN4fNnHa36f5EeLEoiGO3u82KPRFJaWU0DP3t6Hb9/dQt9+hiXnzyWz7z7aEYOzEt2aCJyhKIkgpfM7Hh3Xx57NJJyahuaueOFjdz2/FvUN7dy6ewxfOHsSQztn5vs0ESki0RJBKcDV5nZRoKqIQPc3afHGpkkVXNLK39ctJWfPLmOspoG5k0bzlfnHcPRQwqSHZqIdLEoieD82KOQlOHuPLFqJ9//+5tsKK3lXeML+fUVJ3HSuMJkhyYiMYly++hmADMbCqg+oBdbu7Oaby1YyUtv7eLoIfncdsVJnDt1mG4DFenlotw+eiHwI2AkUAKMA1YD0yK8dh7wUyADuMPdv3fA82OBe4CB4T43uvujh/gZ5AhV1Tdxy5PruOflTRTkZPLt+dO4bPZYMjX+v0haiFI19B1gDvCUu59oZu8BPtbZi8wsA/g5cC5QDCw0swXuvipht38HHnD3X5rZVOBRgv4K0g3cnYeXbON7j61mV20jl7xrLF897xgGaSA4kbQSJRE0ufsuM+tjZn3c/RkzuyXC62YD6919A4CZ3Q/MBxITgQP9w+UBwNuHELscga276/h/f17OC+vKmDl2IHddNZvjR6szmEg6ipIIKsLhJZ4nGHOoBKiN8LpRBOMStSkGTj5gn28BT5jZ9UA+MPdgBzKza4BrAMaOHRvhraU9ra3OvS9v4gePr8GA//rgcVw2e6zmABZJY1ESwXygHvgScDnBN/euGl7iUuBud/+RmZ0C/NbMjnP31sSd3P024DaAWbNmeRe9d9p5q7SGf3toGYs2l/PuyUP47kXHM0odwkTSXpS7hhK//d9zCMfeBoxJWB8dbkt0NTAvfJ+XzSwXKCJolJYu4u48tLiYm/6ykuzMPvzoIydw0cxRuhtIRIAOEoGZvejup5tZNUFd/t6nCDqU9W/npW0WApPCqS63AZcAlx2wzxbgHOBuM5tCcHtq6SF+BulAbUMz//HICv70+jbmHDWIn15yIsPUK1hEErSbCNz99PBnv8M5sLs3m9l1wOMEt4be6e4rzezbwKJwopuvALeb2ZcIks1V7q6qny6y6u0qrvv9EjbtquVLcydz3dkTNS+wiLxDh1VD4S2gK9392MM5eNgn4NEDtt2UsLwKOO1wji3tc3fuX7iVby5YycC8LO771BxOOXpwssMSkRTVYSJw9xYzW2NmY919S3cFJYevvqmFm/6yggcWFXPGpCJu+egMBhfkJDssEUlhUe4aKgRWmtlrJNw26u4XxhaVHJZtFXv43O8Ws6y4kuvPnsgNcyerKkhEOhUlEfxH7FHIEXtxXRnX/2EJzS3O7VfO4typw5Idkoj0EFFuH32uOwKRw3fvy5v41oKVTBxawK8+dhJHaahoETkEUQadmwP8LzAFyCa4A6g2wu2jEjN354ePr+EXz77F3ClD+eklJ5KfE+UiT0Rknyilxq0EfQAeBGYBVwKT4wxKOtfc0sqNf1rOQ4uLuXT2WL4zf5pGCxWRwxKp5HD39UCGu7e4+12EvYElORqbW/ni/Ut5aHExN8ydxHc/dJySgIgctihXBHVmlg0sNbMfANuJmECk69U3tfD5+5bwjzdL+Pf3T+FTZxyV7JBEpIeLUqBfEe53HcHto2OAi+MMSg6urrGZq+9ZyDNrSvjvDx2nJCAiXSLKFcFJwN/cvQr4z5jjkXZU1TfxybsWsmRLeTho3OhkhyQivUSUK4IPAGvN7LdmdoGZ6baUblbX2Mwn7lrI0q0V3HrZTCUBEelSnSYCd/8EMJHgrqFLgbfM7I64A5NAQ3MLn/ntYl7fUs7PLj2R9x0/ItkhiUgvE+nbvbs3mdljBCOE5gEfBD4VZ2ASzCb25T++wQvryvjBxdOVBEQkFp1eEZjZ+WZ2N7COoJH4DmB4zHEJ8P2/v8nflm/nG++bwr+8a0znLxAROQxRrgiuBP4IfMbdG2KOR0L3vbqZXz+/gStPGcenzpiQ7HBEpBeLMtbQpd0RiOzz3NpSbvrLSt5zzBBuumCqppQUkVipY1iKeXNHFdfet4TJw/rxv5fNVI9hEYmdSpkUUlJVz9V3LyI/J4M7r5pFgQaQE5FuoJImRTS1tPLZ3y2mvK6RBz5zCiMG5CU7JBFJE+0mAjNbTnC76EG5+/RYIkpTP35yLUu2VHDrZSdy3KgByQ5HRNJIR1cEF4Q/rw1//jb8eXl84aSn59eW8stn3+LS2WO5YPrIZIcjImmm3UTg7psBzOxcdz8x4akbzWwJcGPcwaWDkup6vvzAUiYPK+CmC6YmOxwRSUNRGovNzE5LWDk14uukE209h2samrn1spnkZWckOyQRSUNRGouvBu40s7aK6wrgk/GFlD5++dxbvLi+jO9ddDyTh/VLdjgikqaidChbDJzQlgjcvTL2qNLA4s3l/PjJtVwwfQQf1fARIpJEUcYaGmZmvwHud/dKM5tqZld3Q2y9Vl1jM19+YCkjBuTy3YuOV89hEUmqKHX9dwOPA223s6wFbogroHTw/cfeZPOuOm7+yAn0z81KdjgikuaiJIIid38AaAVw92agJdaoerGX1pdxz8ub+cRp45lz1OBkhyMiEikR1JrZYMLOZWY2B1A7wWGobWjmqw8tY0JRPl8779hkhyMiAkS7a+jLwALgaDP7JzAE+HCsUfVSP3x8DW9X7uHBz5yiW0VFJGVEuWtoiZm9GzgGMGCNuzfFHlkvs3hzOfe8vIkr5oxj1vhByQ5HRGSvqIPOzQbGh/vPNDPc/d7YouplGppbuPHhZYzon8vX5qlKSERSS6eJwMx+CxwNLGVfI7EDSgQR/eKZt1hXUsNdV71LQ0uLSMqJUirNAqa6e7sjkUr71u6s5hfPrmf+jJG859ihyQ5HROQdotw1tAJNVn9YWlqdrz20jIKcTA0oJyIpK8oVQRGwysxeA/ZOXu/uF8YWVS9x78ubWLq1gls+OoPBBTnJDkdE5KCiJIJvxR1Eb7S7tpEfP7mWMycPYf4MzTEgIqkryu2jzx3uwc1sHvBTIAO4w92/d5B9/oUg2Tjwhrtfdrjvl0p++tRa6hpb+I/3T9FYQiKS0jqaqvJFdz/dzKrZf8pKA9zd+3d0YDPLAH4OnAsUAwvNbIG7r0rYZxLwdeA0dy83s17Rmrq+pIbfvbqFy2aPZZKGlxaRFNfRDGWnhz8PtySbDax39w0AZnY/MB9YlbDPp4Gfu3t5+F4lh/leKeV/Hl1N36wMbpg7KdmhiIh0KvJMY2Y21MzGtj0ivGQUsDVhvTjclmgyMNnM/mlmr4RVSQd772vMbJGZLSotLY0aclK8uK6Mp98s4bqzJ6qBWER6hCjzEVxoZuuAjcBzwCbgsS56/0xgEnAWcClwu5kNPHAnd7/N3We5+6whQ4Z00Vt3vZZW57/+tooxg/L4+Knjkx2OiEgkUa4IvgPMAda6+wTgHOCVCK/bBiROvTU63JaoGFjg7k3uvpFgroMeW5/y0OKtvLmjmn+bdyy5WRpUTkR6hiiJoMnddwF9zKyPuz9D0Nu4MwuBSWY2wcyygUsIRjFN9AjB1QBmVkRQVbQhavCppKahmZufWMvMsQN5//Ejkh2OiEhkUfoRVJhZAfA8cJ+ZlQC1nb3I3ZvN7DqC2c0ygDvdfaWZfRtY5O4Lwufea2arCMYx+mqYdHqcO17YQGl1A7++4iTdLioiPYp1NoSQmeUD9QS3jV4ODADuS1aBPWvWLF+0aFEy3rpdlXVNnP79f3DqxMH8+oooF0siIt3LzBa7+0ELqCgdyhK//d/TZVH1Ir95cQPVDc3cMHdyskMRETlkHXUoO2hHMiJ2KEsXFXWN3PnPTZx/3HCmjNApEZGep6MOZeoSG8EdL2ykpqGZL6rzmIj0UJFmSTGzmcDpBFcEL7r767FG1UNU1zdxz0ubeN/xwzl2uK4GRKRnitKh7CaCtoHBBENS321m/x53YD3BA4uKqW5o5rPvPjrZoYiIHLYoVwSXAye4ez2AmX2PYNrK/4ozsFTX3NLKXf/cyOzxg5g++h2doUVEeowoHcreBnIT1nN4Zw/htPPEqp0Ul+/h6jMmJDsUEZEjEuWKoBJYaWZPErQRnAu8ZmY/A3D3L8QYX8q644UNjB3Ul7lThiU7FBGRIxIlEfw5fLR5Np5Qeo7Xt5SzZEsF3/rAVDL6qBexiPRsURLBYwfOE2Bmx7j7mphiSnn3vryZgpxMPjxrTOc7i4ikuChtBC+E00kCYGZfYf8rhLRSVtPA35Zt5+KZoyjIiXT3rYhISotSkp0F3GZmHwGGAasJZh9LS39cuJXGllauOGVcskMREekSnV4RuPt24O/AKcB44B53r4k5rpTU3NLKfa9s5rSJg5k4VB2vRaR3iNKh7CngZOA44P3ALWZ2c9yBpaKn3yzh7cp6rpgzPtmhiIh0mShtBLe6+5XuXuHuy4FTCW4pTTu/f3ULIwbkMnfK0GSHIiLSZaJUDT1iZuPMbG64KQu4Jd6wUs/OqnpeWFfKh08aTWZGlPwpItIzRKka+jTwEPDrcNNogikm08qfX99Gq8NFM0cnOxQRkS4V5avttcBpQBWAu68D0qpuxN15eHExJ40rZEJRfrLDERHpUlESQYO7N7atmFkm+09Y0+ut2FbFupIaLpo5KtmhiIh0uSiJ4Dkz+39AnpmdCzwI/F+8YaWWh5cUk53Zhwumj0x2KCIiXS5KIrgRKAWWA58BHgXSZj6CppZW/rJ0G+dOHcaAvKxkhyMi0uWiTF7fCtwePtLOS2/toryuiQ/NULWQiPROug+yE48t305BTiZnTC5KdigiIrFQIuhAc0srj6/cwTlThpKTmZHscEREYhE5EZhZ3zgDSUWvbtxNeV0T5x83ItmhiIjEJkqHslPNbBXwZrh+gpn9IvbIUsCjy7fTNzuDs44ZkuxQRERiE+WK4CfAecAuAHd/AzgzzqBSQUur8/jKHbzn2KHkZqlaSER6r0hVQ+6+9YBNLTHEklIWbtpNWU0j71O1kIj0clEmptlqZqcCbmZZwBcJJqfp1R5fuYOczD6qFhKRXi/KFcFnCcYbGgVsA2aE672Wu/PU6p2cPrGIfE1HKSK9XJRSztz98tgjSSFrd9awdfcePvfuickORUQkdlGuCP5pZk+Y2dVmNjD2iFLAU6t3AnCOJqARkTQQZWKayQRjC00DlpjZX83sY7FHlkRPrd7JCaMHMKx/brJDERGJXdS7hl5z9y8Ds4HdwD2xRpVEpdUNLN1awTlThiU7FBGRbhGlQ1l/M/u4mT0GvARsJ0gIvdIzb5bgDnOVCEQkTURpLH6DYGrKb7v7yzHHk3RPrt7JqIF5TBnRL9mhiIh0iyiJ4Ch3T4sZyZpaWnlpfRkfmjkKM0t2OCIi3aLdqiEzuyVcXGBm73hEObiZzTOzNWa23sxu7GC/i83MzWzWIcbfpVZsq6S2sYVTj9aQ0yKSPjq6Ivht+PPmwzmwmWUAPwfOBYqBhWa2wN1XHbBfP4Leyq8ezvt0pVc27AZg9oRBSY5ERKT7tHtF4O6Lw8UZ7v5c4oOgd3FnZgPr3X2DuzcC9wPzD7Lfd4DvA/WHGHuXe3nDLiYPK6CoICfZoYiIdJsot49+/CDbrorwulFA4mB1xeG2vcxsJjDG3f/W0YHM7BozW2Rmi0pLSyO89aFramll0abdzDlqcCzHFxFJVe1WDZnZpcBlwIQD2gT6EfQlOCJm1gf4MRGSirvfBtwGMGvWrFgarpdvq6SusUWJQETSTkdtBG19BoqAHyVsrwaWRTj2NmBMwvrocFubfsBxwLPhHTrDCRqmL3T3RRGO36Ve2bALUPuAiKSfdhOBu28GNgOnHOaxFwKTzGwCQQK4hOAKo+34lQRJBgAzexb412QkAQgaitU+ICLpKErP4jlmttDMasys0cxazKyqs9e5ezNwHfA4wfwFD7j7SjP7tpldeOShdx21D4hIOovSoexWgm/zDwKzgCuByVEO7u6PAo8esO2mdvY9K8ox47AibB84eYISgYikn6iDzq0HMty9xd3vAubFG1b3emNrBQAnjStMciQiIt0vyhVBnZllA0vN7AcEDciREkhPsXxbFUUFOQzrr/YBEUk/UQr0K4AMgvr+WoI7gS6OM6jutmJbJceN6q/xhUQkLXV6RRDePQSwB/jPeMPpfnsaW1hXUs17p2nYaRFJTx11KFsOtNt5y92nxxJRN1u1vYpWh+NGDUh2KCIiSdHRFcEF3RZFEq3YVgnA8UoEIpKmOutQ1ust31bJ4PxsRgzQ/MQikp46bSMws2r2VRFlA1lArbv3jzOw7hI0FA9QQ7GIpK0ojcV752y0oLScD8yJM6juUt/UwrqSGs1PLCJp7ZD6A3jgEeC8mOLpVqu3V9HS6mooFpG0FqVq6KKE1T4Ew0wkfRKZrrC3oXi0EoGIpK8oPYs/kLDcDGzi4DON9TjLt1UyKD+bkWooFpE0FqWN4BPdEUgyrNlZw7HD+6mhWETSWpSqoQnA9cD4xP3dPaWGkj5U7s5bJTVcPHNU5zuLiPRiUaqGHgF+A/wf0BpvON1nR1U9NQ3NTBzWr/OdRUR6sSiJoN7dfxZ7JN1s3c4aACYNLUhyJCIiyRUlEfzUzL4JPAE0tG109yWxRdUN1pUoEYiIQLREcDzBUNRns69qyMP1Hmt9STWFfbMYrDmKRSTNRUkEHwGOcvfGuIPpTutLapg0VO0DIiJRehavAAbGHUh3cnfW7qxh4jBVC4mIRLkiGAi8aWYL2b+NoMfePlpW00jlnia1D4iIEC0RfDP2KLrZupJqAFUNiYgQrWfxc90RSHda33bHkKqGRETScz6CdTtr6JeTydB+umNIRCQt5yNYXxI0FGuMIRGRNJ2PYF1JjRqKRURCaTcfQVV9E2U1DRw9RIlARATScD6C0urgDthh/TUHgYgIpOF8BLtqgg7SRRpaQkQEiNBGYGb3mNnAhPVCM7sz3rDiU1YTXBEMLshOciQiIqkhSmPxdHevaFtx93LgxPhCitcuJQIRkf1ESQR9zKywbcXMBhGtbSElldU0YgaD+ioRiIhAtAL9R8DLZvZguP4R4L/jCyleZTUNFPbNJjPjkO6cFRHptaI0Ft9rZovYN//ARe6+Kt6w4rOrppEiVQuJiOwVqYonLPh7bOGfqKymgcH5umNIRKRN2tWP7KptVEOxiEiCWBOBmc0zszVmtt7MbjzI8182s1VmtszMnjazcXHGA8EVgfoQiIjsE1siMLMM4OfA+cBU4FIzm3rAbq8Ds9x9OvAQ8IO44gGob2qhur5ZbQQiIgnivCKYDax39w3hfMf3c8DQFO7+jLvXhauvAKNjjIfdtUGvYk1YLyKyT5yJYBSwNWG9ONzWnquBx2KMZ2+vYlUNiYjskxIdw8zsYwSjmr67neevAa4BGDt27GG/T9s4Q2osFhHZJ84rgm3AmIT10eG2/ZjZXOAbwIXu3nCwA7n7be4+y91nDRky5LAD2qJ0s2oAAAoQSURBVHtFoNtHRUT2ijMRLAQmmdkEM8sGLgEWJO5gZicCvyZIAiUxxgIEw0sAFPXTFYGISJvYEoG7NwPXAY8Dq4EH3H2lmX3bzC4Md/shUAA8aGZLzWxBO4frErtqGsjLyqBvdkrUiImIpIRYS0R3fxR49IBtNyUsz43z/Q9UVtOgqwERkQOkVc/iXbWNGl5CROQAaZUIyjTgnIjIO6RZItDwEiIiB0qbRNDa6uzWgHMiIu+QNomgYk8TLa2uNgIRkQOkTSJom6u4qJ8SgYhIorRJBHs7k+WrakhEJFEaJYLgikAjj4qI7C9tEsHeqiE1FouI7CdtEsHIgXm8d+owBvZVIhARSZQ2g+68d9pw3jtteLLDEBFJOWlzRSAiIgenRCAikuaUCERE0pwSgYhImlMiEBFJc0oEIiJpTolARCTNKRGIiKQ5c/dkx3BIzKwU2HyYLy8CyrownO6m+JNL8SeX4j8y49x9yMGe6HGJ4EiY2SJ3n5XsOA6X4k8uxZ9cij8+qhoSEUlzSgQiImku3RLBbckO4Agp/uRS/Mml+GOSVm0EIiLyTul2RSAiIgdQIhARSXNpkwjMbJ6ZrTGz9WZ2Y7Lj6YyZjTGzZ8xslZmtNLMvhtsHmdmTZrYu/FmY7FjbY2YZZva6mf01XJ9gZq+Gv4M/mllKTxdnZgPN7CEze9PMVpvZKT3l/JvZl8K/mxVm9gczy031829md5pZiZmtSNh20PNtgZ+Fn2WZmc1MXuR7Yz1Y/D8M/36WmdmfzWxgwnNfD+NfY2bnJSfqQFokAjPLAH4OnA9MBS41s6nJjapTzcBX3H0qMAe4Noz5RuBpd58EPB2up6ovAqsT1r8P/MTdJwLlwNVJiSq6nwJ/d/djgRMIPkvKn38zGwV8AZjl7scBGcAlpP75vxuYd8C29s73+cCk8HEN8MtuirEjd/PO+J8EjnP36cBa4OsA4f/yJcC08DW/CMuppEiLRADMBta7+wZ3bwTuB+YnOaYOuft2d18SLlcTFEKjCOK+J9ztHuCDyYmwY2Y2Gng/cEe4bsDZwEPhLikbO4CZDQDOBH4D4O6N7l5BDzn/BNPQ5plZJtAX2E6Kn393fx7YfcDm9s73fOBeD7wCDDSzEd0T6cEdLH53f8Ldm8PVV4DR4fJ84H53b3D3jcB6gnIqKdIlEYwCtiasF4fbegQzGw+cCLwKDHP37eFTO4BhSQqrM7cAXwNaw/XBQEXCP0Wq/w4mAKXAXWH11h1mlk8POP/uvg24GdhCkAAqgcX0rPPfpr3z3RP/pz8JPBYup1T86ZIIeiwzKwAeBm5w96rE5zy49zfl7v81swuAEndfnOxYjkAmMBP4pbufCNRyQDVQCp//QoJvnBOAkUA+76yy6HFS9XxHYWbfIKjuvS/ZsRxMuiSCbcCYhPXR4baUZmZZBEngPnf/U7h5Z9slcPizJFnxdeA04EIz20RQDXc2QX37wLCqAlL/d1AMFLv7q+H6QwSJoSec/7nARncvdfcm4E8Ev5OedP7btHe+e8z/tJldBVwAXO77Om6lVPzpkggWApPCuyayCRppFiQ5pg6Fdeq/AVa7+48TnloAfDxc/jjwl+6OrTPu/nV3H+3u4wnO9T/c/XLgGeDD4W4pGXsbd98BbDWzY8JN5wCr6AHnn6BKaI6Z9Q3/jtpi7zHnP0F753sBcGV499AcoDKhCillmNk8girSC929LuGpBcAlZpZjZhMIGr1fS0aMALh7WjyA9xG02r8FfCPZ8USI93SCy+BlwNLw8T6CuvangXXAU8CgZMfayec4C/hruHwUwR/7euBBICfZ8XUS+wxgUfg7eAQo7CnnH/hP4E1gBfBbICfVzz/wB4I2jSaCK7Kr2zvfgBHcCfgWsJzgDqlUjH89QVtA2//wrxL2/0YY/xrg/GTGriEmRETSXLpUDYmISDuUCERE0pwSgYhImlMiEBFJc0oEIiJpTolAejQze9bMYp8Q3My+EI5AmpI9Q7tKOOLq55Mdh3QvJQJJWwm9bKP4PHCuBx3jerOBBJ9V0ogSgcTOzMaH36ZvD8fIf8LM8sLn9n6jN7OicFgKzOwqM3skHIN+k5ldZ2ZfDgeAe8XMBiW8xRVmtjQce392+Pr8cHz418LXzE847gIz+wdBR6UDY/1yeJwVZnZDuO1XBJ2xHjOzLx2wf4aZ3Rzuv8zMrg+3nxO+7/Iwjpxw+yYz+58w3kVmNtPMHjezt8zss+E+Z5nZ82b2t3Cs+l+ZWZ/wuUvDY64ws+8nxFFjZv9tZm+E52dYuH2ImT1sZgvDx2nh9m+FcT1rZhvM7Avhob4HHB3G90MzGxHG0nZ+zzjsPwRJXcnujadH738A4wkG3JoRrj8AfCxcfpawVyhQBGwKl68i6JXZDxhCMILmZ8PnfkIwCF/b628Pl88EVoTL3014j4EEvcrzw+MWc5AewcBJBL1U84ECYCVwYvjcJqDoIK/5HME4RJnh+iAgl6A36eRw270J8W4CPpfwOZYlfMad4fazgHqC5JNBMKb9hwkGkNsS7psJ/AP4YPgaBz4QLv8A+Pdw+ffA6eHyWIIhSwC+BbxE0OO4CNgFZIW/qxUJn+8rhD3xw1j6JfvvSY+ufxzKpbHIkdjo7kvD5cUEBU5nnvFgLoZqM6sE/i/cvhyYnrDfHyAYD97M+lswC9R7CQa++9dwn1yCghDgSXc/cNx7CIb1+LO71wKY2Z+AM4DXO4hxLsGwAc1hDLvN7ITw864N97kHuJZgaG7YN87VcqAg4TM22L4ZrF5z9w1hHH8IY2sCnnX30nD7fQTJ7xGgEfhr+NrFwLkJ8U0NhhwCoL8FI9oC/M3dG4AGMyvh4ENqLwTutGAAxEcSfofSiygRSHdpSFhuAfLC5Wb2VVHmdvCa1oT1Vvb/2z1wnBQnGIvmYndfk/iEmZ1MMKR0MiV+jgM/Y9vnOthn6kiTu7ft05JwnD7AHHevT9w5TAwH/k7eUR6EyfVMgkmG7jazH7v7vZ3EIj2M2ggk2TYRVMnAvpExD9VHAczsdIJRKCuBx4Hrw9E3MbMTIxznBeCD4aid+cCHwm0deRL4TFvDc9h2sQYYb2YTw32uAJ47xM8024LRcvsQfL4XCQaMe3fYlpIBXBrhuE8A17etmNmMTvavJqiqatt/HEGV1e0Es80lfW5g6XpKBJJsNwOfM7PXCeqqD0d9+PpfsW8e3u8Q1HkvM7OV4XqHPJga9G6CAvdV4A5376haCILCcUv4Pm8Al4Xfvj8BPGhmywm+6f/qED/TQuBWgilKNxJUWW0nmBznGeANYLG7dzaU9BeAWWFD9irgsx3t7O67gH+GDcM/JGiveCM8vx8lmFdCehmNPiqSYszsLOBf3f2CZMci6UFXBCIiaU5XBCIiaU5XBCIiaU6JQEQkzSkRiIikOSUCEZE0p0QgIpLm/j9ITYubJQZNnwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca_curve(X[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3319: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reduced shape: (2131, 1537)\n",
      "recovered shape: (2131, 66564)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 6850.76628218, -3250.48454657,  2528.35591362, ...,\n",
       "         -141.63793441,   359.66428155,   244.38698818],\n",
       "       [-7379.97520022,  -212.74894326,   874.8223641 , ...,\n",
       "          454.40975551,   338.12137599,   363.27936405],\n",
       "       [10129.98991647, -3191.04580719,   138.6494128 , ...,\n",
       "         -174.51660728,   -77.01994593,   380.1207462 ],\n",
       "       ...,\n",
       "       [ 7009.99848315, -3321.77563341,  2545.40931605, ...,\n",
       "          232.44406623,    90.12536548,  -171.45441427],\n",
       "       [ 8314.82627773, -3129.38762905,  1264.6657795 , ...,\n",
       "          189.62604529,  -161.41497948,   166.79855644],\n",
       "       [-1325.13954579,   439.86960396, -4043.67235807, ...,\n",
       "           45.47816145,  -535.93468863,  -323.31143148]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd3wc9Z3/8dfHsizJlmTZltzkIndwARtkY0xNaAYCJiEkEEILgUBCSM9xRy7h0n4cqeTCXQKEEkIgdBxKgCSUxATce5W75SK5SJZtdX1+f8zIWRyXcRmtpH0/H4997Mzs7O5nR6v57HyruTsiIpK6OiQ7ABERSS4lAhGRFKdEICKS4pQIRERSnBKBiEiK65jsAA5Xfn6+FxUVJTsMEZE2ZdasWVvdvWB/j7W5RFBUVMTMmTOTHYaISJtiZmsP9JiKhkREUpwSgYhIilMiEBFJcUoEIiIpTolARCTFxZYIzOwhMyszs4UHeNzM7BdmVmJm883spLhiERGRA4vziuARYPJBHr8QGBbebgb+L8ZYRETkAGLrR+Du75hZ0UF2mQL81oNxsN8zszwz6+Pum+KKSUSktXB3KqvrP3Cr2BPc76ypp7HRaXJocqd5uoBzju/Fif3zjnksyexQVgisT1jfEG77l0RgZjcTXDUwYMCAFglORORINDU5W3fVsmVnLWVVNZRV1VKWuFxVS/nOGsp31VLfGH0+GDPomZvZ7hJBZO5+P3A/QHFxsWbSEZGkqW1oZGNFDRsrqindUc2Giuq9y6UV1WyqrN7vCb5b53R65mTSMzeDIQU96JmTSX52J7p17kTXrHS6dk6na1Y6eVnp5Galk57WgQ4GZhb7Z0pmIigF+ies9wu3iYgkjbuzdVcda7ftZs22Paxrvt++h9KKasqraj+wvxn0ysmkb17wa/2iMX3om5dJr9xMeuZk0DM3OOFndExL0ic6tGQmgqnAbWb2JHAKUKn6ARFpCe7O5p01rNm6Z+8Jf+223awN73fXNe7dt4NBYbcsBnTvzIdGFFCY15nCblkU5gW33l0z6dSxbbfEjy0RmNkTwNlAvpltAL4DpAO4+6+AV4CLgBJgD3BDXLGISGqqb2xi7bY9lJTtYmX5LlaW7aIkvE882aenGf27d2Zg985MGNSdoh6dGZjfhaIeXSjMy2rzJ/pDibPV0FWHeNyBL8T1/iKSOmrqGykp28WKsipKynbtva3dtoeGpn+W1/fOzWRoz2yuKO7PkIIuDMrPZmCPzvTNyyKtQ/xl8a1Vm6gsFhGBoEintKKapZuqWLaliiWbdrJ0cxWrt+6mMTzhp3UwBvbozNCCbM4f1ZuhBdkM7ZnNkJ7ZZGfolLc/Oioi0irtrm1g6ebgRL90U1WwvKmKqtqGvfv0757Fcb1zuXB0b47rncvwXtkM7NGl3RflHGtKBCKSdLtqG1hUWsnCjTtZWFrJgtJKVpbvIuxHRU5GR47rk8Nl4wo5rk8Ox/XOYXivHHIy05MbeDuhRCAiLaqqpp6FpTtZtDE44S8orWT11t17T/q9cjMYU9iVj5zQh1F9uzKyby59u2a2SHv6VKVEICKxaWhsYtmWKuasq2DOugrmrt/ByvLdex/v0zWT0YVduWxsIaMLcxld2JWeOZlJjDg1KRGIyDFTtrOGOesrwhP/DuZvqKS6Pmim2aNLJ8YNyAtO+v26MqawK/nZGUmOWECJQESOUGOTs2TTTqav3s7sdTuYs66C0opqIGiXP7JPLp8c359xA/IY178b/btnqXinlVIiEJFIahsamb+hkumrtwcn/7U79rbg6ds1k3EDu3HDaUWMG9CNUX1zyUxvvUMqyAcpEYjIfu2qbWDW2h3MCE/8czdUUNfQBMCwntlcMrYvE4q6M35QdwrzspIcrRwNJQIRAYLeubPW7mBayVamrdzGgg0VNHnQQWt031yunTiQ8YO6M76oO927dEp2uHIMKRGIpKiGxibmbajkHyu3Mq1kG7PW7aCuoYmOHYyx/fP4woeGcsqgHowbkEcX9cht1/TXFUkR7s6yLVX8fcVW3l25jemrt7MrLOMf2SeX604dyKQh+Ywf1F1DMaQY/bVF2rHK6nqmlWzl7WXlvL28nM07awAYlN+FKWP7MmlIPqcO6aGinhSnRCDSjjQ1OYs37eStZWW8vbyc2esqaGxycjI7csawfM4aXsDpwwpUuSsfoEQg0sZVVtfz9vJy3lpaxjsrytm6qw6AMYVdufWsIZw1ooBx/fPomKaB2GT/lAhE2qD12/fwlyVbeGPJFt5ftZ2GJqdb53TOHF7AWcMLOGNYAQU56rUr0SgRiLQBTU3Owo2VvLF4C28s3sLSzVUADO2ZzWfPGMx5I3sytn+3lJ5cRY6cEoFIK1Xf2MS7K7fx2qLN/GXJFrbsrKWDQXFRd+686HjOHdmLQfldkh2mtAOREoGZnQ4Mc/eHzawAyHb31fGGJpJ66hqamFaylZcXbOKNxVuorK6nc6c0zhpewLnH9+LDx/Wkm1r4yDF2yERgZt8BioERwMMEE9D/Djgt3tBEUkNNfSN/X7GVVxYGJ/+qmgZyMjty3sheXDS6D6cPy9e4PRKrKFcEHwXGAbMB3H2jmeXEGpVIO1fb0Mhby8p5dcEm/rykjF21DXTNSmfyqN5cNKYPk4b2IKOjTv7SMqIkgjp3dzNzADNToaTIEWhqct5bvY2pczfyyoJN7KxpIK9zOheP6cNFJ/Th1ME9NNeuJEWURPCUmf0ayDOzm4DPAA/EG5ZI++AedPB6ce5Gps7dyOadNXTplMYFo3ozZVwhk4b0IF3t+yXJDpkI3P3HZnYesJOgnuDb7v5G7JGJtGHrt+9h6ryNvDCnlBVlu+jYwTh7RAF3Xnw85x7fi6xOKvaR1iNKZfEg4G/NJ38zyzKzIndfE3dwIm3JnroGXl2wmadmruf91dsBGF/Uje9fNpqLx/RRax9ptaIUDT0NTEpYbwy3jY8lIpE2xN2ZvW4HT8/cwEvzN7GrtoGiHp35xgUjmDK2L/26dU52iCKHFCURdHT3uuYVd68zM/20kZRWtrOG5+aU8tTM9awq303nTmlcPKYPVxT3Z3xRN83NK21KlERQbmaXuvtUADObAmyNNyyR1qexyXlrWRm/f38dby0vp7HJGV/UjVvOGsLFY/po8hZps6J8c28BHjezXwIGrAeujTUqkVakrKqGp2as54np6ymtqKYgJ4PPnTmYj5/cj8EF2ckOT+SoRWk1tBKYaGbZ4fqu2KMSSTJ35x8rt/G799fy+qItNDQ5pw/N51sXB2P8qMmntCdRWg1lAJcDRUDH5rJPd/9urJGJJEHlnnqenrWe37+/jlVbd5PXOZ0bTiviqgkD9Otf2q0oRUMvApXALKA23nBEkqOkbBePvLuaZ2eVUl3fyEkD8vjpJ07kojF9NM6PtHtREkE/d58ceyQiLczd+duKrTw0bTVvLSunU1oHpozty/WnFTGqb9dkhyfSYqIkgnfNbIy7L4g9GpEWUF3XyHNzNvDwtDWUlO0iPzuDr543nE+dMoD8bM3qJaknSiI4HbjezFYTFA0Z4O5+QqyRiRxj5VW1PPLuah5/fx0Ve+oZXZjLTz9xIhef0EcjfUpKi5IILow9CpEYrdu2h/v/tpKnZm6gvrGJC0b25sYzBlE8UB2/RCBa89G1AGbWE8iMPSKRY2TRxkp+9fYqXp6/kY4dOnD5yYXcdMZgtf4R2UeU5qOXAj8B+gJlwEBgCTAq3tBEDp+7849V2/i/t1bytxVbyc7oyE1nDuYzpw2iV65+x4jsT5Sioe8BE4E/u/s4M/sQ8Ol4wxI5PM0tgH7+5+XMXldBfnYG35w8gqtPGUjXrPRkhyfSqkVJBPXuvs3MOphZB3d/08x+HuXFzWwycC+QBjzo7nfv8/gA4FEgL9znDnd/5fA+gqQyd+edMAHMWVdB366ZfO+y0Vxxcj+1/xeJKEoiqAiHl3iHYMyhMmD3oZ5kZmnAfcB5wAZghplNdffFCbt9C3jK3f/PzEYCrxD0YBY5KHfn7eXl/PzPK5i7PkgAP/joaD5+cj+1ABI5TFESwRSgBvgKcDXQFYgyvMQEoMTdVwGY2ZPhayUmAgdyw+WuwMZoYUuq2jcBFOZl8cOPjuHjJ/fTfL8iRyhKq6HEX/+PHsZrFxKMVNpsA3DKPvvcBbxuZl8EugDn7u+FzOxm4GaAAQMGHEYI0p7MWLOde/60lBlrdigBiBxDB0wEZvZ3dz/dzKoIfrnvfYigQ1nuAZ56OK4CHnH3n5jZqcBjZjba3ZsSd3L3+4H7AYqLi30/ryPt2NLNO/nRn5bxl6VlFORk8L3LRvPJ4v5KACLHyAETgbufHt7nHOFrlwL9E9b7hdsS3QhMDt/nH2aWCeQTNFOVFLd++x5+9sZynp9bSnZGR745eQTXTyqicydNACNyLB30Pyqs8F3k7scdwWvPAIaZ2SCCBHAl8Kl99lkHnAM8YmbHE3RYKz+C95J2ZOuuWn751xIef38tHcy4+czB3HrWEPI6a4ZUkTgcNBG4e6OZLTOzAe6+7nBe2N0bzOw24DWCpqEPufsiM/suMDOc+vJrwANm9hWC4qfr3V1FPymqpr6Rh6et4b43S6iub+ST4/tz+4eH0burOoKJxCnKNXY3YJGZTSeh2ai7X3qoJ4Z9Al7ZZ9u3E5YXA6dFjlbaJXfn5QWbuPvVpWzYUc25x/fk3y86niEaCkKkRURJBP8ZexSSsuaur+D7Ly1m5todHNc7h8c/ewqnDc1PdlgiKSVK89G3WyIQSS2bKqv571eX8sLcjeRnZ3D3x8ZwRXF/0jpoNFCRlhZl0LmJwP8AxwOdCMr7dx+j5qOSYuoamnho2mp+8ZcVNDQ5nz97CJ//0FCyM9QSSCRZovz3/ZKgxc/TQDFwLTA8zqCkfXq3ZCv/+eJCVpbv5tzje/GdS0bSv3vnZIclkvIi/Qxz9xIzS3P3RuBhM5sD/Hu8oUl7sbmyhh+8soQ/ztvIgO6d+c11xZxzfK9khyUioSiJYI+ZdQLmmtk9wCZAXTrlkOobm3hk2hp+/ufl1Dc5Xz53GLecNUSjgoq0MlESwTUEJ/7bCAae6w9cHmdQ0vbNXV/BHc/OZ+nmKs45riffuWQUA3qoGEikNYqSCE4GXnb3ncB/xRyPtHG7axv4yevLefjd1fTOzeT+a07m/FG9kx2WiBxElERwCfAzM3sH+APwJ3dviDcsaYveWlbGnc8vpLSimmtPHcg3LhhBTqZmBxNp7aL0I7jBzNKBCwlGC73PzN5w98/GHp20Cdt31/G9lxbz/JxShhR04ZlbTqW4qHuywxKRiKK2Gqo3s1cJxgPKAi4DlAiEl+Zv5NsvLqKqpp7bzxnGFz40RDOEibQxUTqUXQh8EjgbeAt4EPhErFFJq7djdx3/+eJCXpq/iRP753HP5ScwoveRjlguIskU5YrgWoK6gc+5e23M8Ugb8NelW/i3ZxdQsaeOr58/nFvOGkLHNLUoFmmrotQRXNUSgUjrV1VTz/dfWsIfZq7nuN45PHLDeEb17ZrssETkKGmAF4nkHyu38fWn57Gpsppbzx7Cl88dproAkXZCiUAOqr6xiZ++sZxfvb2Sgd078/Qtp3LyQLUIEmlPlAjkgNZt28MXn5zDvPUVXDm+P9++ZKTmCxZphw74X21mCwiai+6Xu58QS0TSKrw4t5Q7n1+IGdz3qZO4+IQ+yQ5JRGJysJ93HwnvvxDePxbeXx1fOJJsu2sb+M7URTwzawMnD+zGvVeOpV83jREk0p4dMBG4+1oAMzvP3cclPHSHmc0G7og7OGlZSzbt5AuPz2bNtt3cfs4wbv/wUDULFUkBUQp8zcxOc/dp4cokNAx1u/PMrA1864UF5Gam8/ubJjJxcI9khyQiLSRKIrgReMjMmhuMVwCfiS8kaUk19Y381x8X88T0dZw6uAe/uGocBTkZyQ5LRFpQlA5ls4ATmxOBu1fGHpW0iPXb93Dr47NYWLqTW88ewtfOG66iIJEUFGWsoV7AD4G+7n6hmY0ETnX338QencTmzaVlfPkPc2ly54FrizlvpKaOFElVUX7+PQK8BvQN15cDX44rIImXu3PfmyV85tEZFOZl8dIXT1cSEElxURJBvrs/BTQBhJPSNMYalcSiuq6R25+cy49eW8YlJ/Tluc9PYmCPLskOS0SSLEpl8W4z60HYuczMJgKqJ2hjNlVWc/NvZ7FwYyXfnDyCW88agpklOywRaQWiJIKvAlOBIWY2DSgAPh5rVHJMzVq7g889NovqugYeuKaYc1UUJCIJorQamm1mZwEjAAOWuXt97JHJMfHc7A3c8ewCenfN5Pc3ncLwXpo8RkQ+KOoIYhOAonD/k8wMd/9tbFHJUXN3/uevJfz0jeWcOrgH/3v1SXTr0inZYYlIKxSl+ehjwBBgLv+sJHZAiaCVqm9s4s7nF/DUzA18bFwhd19+Ap06qn+AiOxflCuCYmCkux9wJFJpPapq6vn847P524qt3P7hoXzlvOGqFBaRg4qSCBYCvYFNMcciR2lzZQ3XPzydFWW7uOfyE/jE+P7JDklE2oAoiSAfWGxm04G9k9e7+6WxRSWHbWX5Lq558H0qq+t56PrxnDW8INkhiUgbESUR3BV3EHJ0FpZWcu1D0+lg8IfPncroQk0oLyLRRWk++nZLBCJH5v1V2/jsozPJzUrnsRsnMLggO9khiUgbc7CpKv/u7qebWRUfnLLSAHf33Nijk4N6c2kZt/xuFv26ZfHYjafQNy8r2SGJSBt0sBnKTg/v1QOpFZo6byNf/cNcju+TyyM3jKdHtuYQEJEjE7lxuZn1NLMBzbeIz5lsZsvMrMTM9ju1pZl9wswWm9kiM/t91HhS2bOzNvClJ+dw0sBu/P6mU5QEROSoROlQdinwE4JhqMuAgcASYNQhnpcG3AecB2wAZpjZVHdfnLDPMODfgdPcfYeZ9TzSD5Iqnpm1gW88M4/ThuTzwLXFZHVKS3ZIItLGRbki+B4wEVju7oOAc4D3IjxvAlDi7qvcvQ54Epiyzz43Afe5+w4Ady+LHHkKenrm+r1J4MHrlARE5NiIkgjq3X0b0MHMOrj7mwS9jQ+lEFifsL4h3JZoODDczKaZ2XtmNjlS1CnoqZnr+eaz8/cmgcx0JQEROTai9COoMLNs4B3gcTMrA3Yfw/cfBpwN9APeMbMx7l6RuJOZ3QzcDDBgQKTqiXblqZnr+bdn53P60KA4SElARI6lKFcEU4Bq4CvAn4CVwCURnlcKJI5x0C/clmgDMNXd6919NcE0mMP2fSF3v9/di929uKAgtXrMvji3VElARGJ1yETg7rvdvdHdG9z9UXf/RVhUdCgzgGFmNsjMOgFXEkxwk+gFgqsBzCyfoKho1WF9gnbsL0u28LWn5jG+qDv3X6MkICLxOFiHsv12JCNihzJ3bzCz2wgmvk8DHnL3RWb2XWCmu08NHzvfzBYTDHH9jYhJpt17d+VWbn18NiP75vIbVQyLSIysrY0uXVxc7DNnzkx2GLGas24Hn37wffrmZfHU507VhDIictTMbJa777ehT6QZyszsJOB0giuCv7v7nGMYnyRYtrmK6x+eQY/sDH732VOUBEQkdoesIzCzbwOPAj0IhqR+xMy+FXdgqWjDjj1c85v3yUzvwOOfPYVeuZnJDklEUkCUK4KrgRPdvQbAzO4mmLby+3EGlmoq99Rz/cMzqK5v5JlbJtG/e+dkhyQiKSJK89GNQOJP0wz+tRmoHIXahkZuemwma7ft5tfXnMyI3hrnT0RaTpQrgkpgkZm9QVBHcB4w3cx+AeDut8cYX7vX1OR87al5TF+9nXuvHMukIfnJDklEUkyURPB8eGv2VjyhpKa7/7SUl+Zv4t8mH8eUsfuOwCEiEr8oieDVfQeDM7MR7r4spphSxmPvreX+d1ZxzcSB3HLW4GSHIyIpKkodwd/M7BPNK2b2NT54hSBH4N2Srdw1dREfGlHAXZeOwsySHZKIpKgoVwRnA/eb2RVAL4K5CCbEGVR7t2brbm59fDaD87vwi6vGkdZBSUBEkifKWEObCAabOxUoAh51910xx9Vu7ayp57O/nYkZPHhdMTmZ6ckOSURSXJQZyv5M0IR0NMFoor8xs3fc/etxB9feNDY5X3piDqu37uaxz0xgYI8uyQ5JRCRSHcEv3f1ad69w9wXAJIImpXKY7nltKW8uK+euS0cxaaiaiYpI6xClaOgFMxtoZueGm9KBn8cbVvvz6oJN/PrtVVx9ygCumTgw2eGIiOwVZayhm4BngF+Hm/oRzCMgEa0q38U3npnPif3z+PYlI5MdjojIB0QpGvoCcBqwE8DdVwA94wyqPamua+Tzj88mPc3436tPIqOj5hUQkdYlSiKodfe65hUz68gHJ6yRA3B37nxhAcu2VHHvleMozMtKdkgiIv8iSiJ428z+A8gys/OAp4E/xhtW+/DE9PU8N7uUL50zjDOHp9ZcyyLSdkRJBHcA5cAC4HPAK4DmIziEJZt2ctcfF3Hm8AJu//CwZIcjInJAh+xH4O5NwAPhTSKoqW/k9ifm0DUrnZ994kQ6qOewiLRikaaqlMPzg5eXsKJsF4/dOIEe2RnJDkdE5KCiFA3JYXhj8RYee28tN50xiDOGqV5ARFq/yInAzDR34iFs2VnDN5+Zx6i+uXz9ghHJDkdEJJIoHcommdliYGm4fqKZ/W/skbUx7s7Xn55HdX0j9145Tv0FRKTNiHJF8DPgAmAbgLvPA86MM6i26PfT1/G3FVv51sUjGdozO9nhiIhEFqloyN3X77OpMYZY2qz12/fww5eXcPrQfK4+ZUCywxEROSxRWg2tN7NJgJtZOvAlgslphKBI6I7n5gNw9+VjNNOYiLQ5Ua4IbiEYb6gQKAXGhutCUCQ0rWQb/3Hx8fTrpvp0EWl7olwRmLtfHXskbVBikdCnJqhISETapihXBNPM7HUzu9HM8mKPqI0IBpRbCKhISETatigT0wwnGFtoFDDbzF4ys0/HHlkr99L8TbyzvJyvXzBCRUIi0qZFbTU03d2/CkwAtgOPxhpVK1dZXc93X1rMmMKuXHtqUbLDERE5KlE6lOWa2XVm9irwLrCJICGkrB+/toxtu2r5fx8bQ5oGlBORNi5KZfE8gqkpv+vu/4g5nlZv7voKfvf+Wq6fVMTowq7JDkdE5KhFSQSD3V0zkgGNTc5/PLeAXjmZfO18jSUkIu3DAROBmf3c3b8MTDWzf0kE7n5prJG1Qn+YsZ7Fm3byy0+NIztDI3iLSPtwsLPZY+H9j1sikNZuZ009P3l9GROKunPxmD7JDkdE5Jg5YCJw91nh4lh3vzfxMTP7EvB2nIG1Nv/zlxVs31PHo5eMVJ8BEWlXojQfvW4/264/xnG0aqu37uaRd9dwxcn9VEEsIu3OweoIrgI+BQwys6kJD+UQ9CVIGT94eQkZHdM02YyItEsHqyNo7jOQD/wkYXsVMD/Ki5vZZOBeIA140N3vPsB+lwPPAOPdfWaU124pM9ds589LtvCNC0bQMycz2eGIiBxzB6sjWAusBU49khc2szTgPuA8YAMww8ymuvviffbLIRja+v0jeZ84uTv3/GkZBTkZfOa0QckOR0QkFlF6Fk80sxlmtsvM6sys0cx2RnjtCUCJu69y9zrgSWDKfvb7HvDfQM1hRd4C3l5ezvQ127n9w0PJ6qSpJ0WkfYpSWfxL4CpgBZAFfJbgl/6hFAKJM5ttCLftZWYnAf3d/eWDvZCZ3WxmM81sZnl5eYS3PnpNTc6PXltG/+5ZfHK8hpgWkfYr6qBzJUCauze6+8PA5KN9YzPrAPwU+FqE97/f3YvdvbigoOBo3zqSVxduZtHGnXzl3OF06hjpMImItElRusfuMbNOwFwzu4egAjnKmbEU6J+w3i/c1iwHGA28FbbL703Qi/nSZFcYuzv/89cVDCnowpSxhYd+gohIGxblhH4NQauf24DdBCf3yyM8bwYwzMwGhYnkSmBvM1R3r3T3fHcvcvci4D0g6UkA4M1lZSzdXMXnzx6q0UVFpN075BVB2HoIoBr4r6gv7O4NZnYb8BpBInnI3ReZ2XeBme4+9eCvkBzuzi//WkJhXhaXju2b7HBERGJ3sA5lC4ADjjrq7icc6sXd/RXglX22ffsA+559qNdrCe+t2s7sdRV8b8oo0tNUNyAi7d/Brgg+0mJRtCL3v7OS/OwMrijuf+idRUTagUN1KEspa7bu5q3l5dz+4WFkpqvfgIikhkPWEZhZFf8sIuoEpAO73T03zsCS4bH31pJmxqdOUb8BEUkdUSqLc5qXLWjnOQWYGGdQybCnroGnZq7nwjF96JWrMYVEJHUcVm2oB14ALogpnqR5fk4pVTUNXD9pYLJDERFpUVGKhj6WsNoBKKYVjgt0NNydx/6xllF9czlpQLdkhyMi0qKi9Cy+JGG5AVjD/gePa7Pmb6hk6eYqfvjRMZp9TERSTpQ6ghtaIpBkenrWejLTO/CREzUXsYiknihFQ4OALwJFifu7+6XxhdVyauobeXHuRiaP6k1uZnqywxERaXFRioZeAH4D/BFoijeclvf64i1U1TTwCXUgE5EUFSUR1Lj7L2KPJEleXbCJnjkZTBzcI9mhiIgkRZREcK+ZfQd4Haht3ujus2OLqoXU1Dfy9vJyPnZSIR00yqiIpKgoiWAMwVDUH+afRUMerrdp00q2sqeukfNH9k52KCIiSRMlEVwBDA7nHW5XXlu0mZyMjioWEpGUFqVn8UIgL+5AWpq789elZZx9XE9NRSkiKS3KFUEesNTMZvDBOoI23Xx0+ZZdbN1VxxnD8pMdiohIUkVJBN+JPYokmFayFYBJQ1QsJCKpLUrP4rdbIpCW9u7KbQzs0Zl+3TonOxQRkaQ6ZOG4mVWZ2c7wVmNmjWa2syWCi4u7M2PNdk4Z1D3ZoYiIJF1Kzkewfns1ldX1nNi/3dWBi4gctpScj2BBaSUAJxQqEYiIpOR8BAtKK0lPM4b3zk52KCIiSZeS8xEsKK1gRO8cMjpqgnoRkZSbj8DdWVi6k4vGaO4BERGI1mroUTPLS1jvZmYPxRtWfFI1OKIAAArpSURBVDbsCCqKxxR2TXYoIiKtQpTK4hPcvaJ5xd13AOPiCyleSzdXAXB8n5xD7CkikhqiJIIOZrZ3Rncz6060uoVWafmWIBEM66VEICIC0U7oPwH+YWZPh+tXAD+IL6R4Ld9SRd+umWRntNlcJiJyTEWpLP6tmc3kn/MPfMzdF8cbVnyWb9mlqwERkQSRfhaHJ/42e/Jv5u6s3babiYM1tISISLOUGoi/Yk89e+oaKczLSnYoIiKtRkolgtKKagD6dVMiEBFpllKJYMOOIBEU5mnoaRGRZimVCJqvCAp1RSAisldqJYId1WSlp9Gtc3qyQxERaTVSKhFsqaqhd9dMgmkVREQEUiwRbK2qJT+7U7LDEBFpVVIrEeyqJT87I9lhiIi0KimWCOqUCERE9hFrIjCzyWa2zMxKzOyO/Tz+VTNbbGbzzewvZjYwrljqGpqorK5XIhAR2UdsicDM0oD7gAuBkcBVZjZyn93mAMXufgLwDHBPXPFs210LQH6O6ghERBLFeUUwAShx91XuXgc8yT5TXLr7m+6+J1x9D+gXVzAVe+oB6N5ZiUBEJFGciaAQWJ+wviHcdiA3Aq/u7wEzu9nMZprZzPLy8iMKprq+EYCsTpqnWEQkUauoLDazTwPFwI/297i73+/uxe5eXFBQcETvUVMXJILMdCUCEZFEcc7OUgr0T1jvF277ADM7F7gTOMvda+MKpqYhvCJQIhAR+YA4rwhmAMPMbJCZdQKuBKYm7mBm44BfA5e6e1mMsVBd1wToikBEZF+xJQJ3bwBuA14DlgBPufsiM/uumV0a7vYjIBt42szmmtnUA7zcUdtbR6BEICLyAbFO3OvurwCv7LPt2wnL58b5/olq6pvrCFpFtYiISKuRMmfFvYlArYZERD4gZRLBgO6duXB0bxUNiYjsI9aiodbk/FG9OX9U72SHISLS6qTMFYGIiOyfEoGISIpTIhARSXFKBCIiKU6JQEQkxSkRiIikOCUCEZEUp0QgIpLizN2THcNhMbNyYO0RPj0f2HoMw2lvdHwOTMfm4HR8Dqy1HJuB7r7fCV3aXCI4GmY2092Lkx1Ha6Xjc2A6Ngen43NgbeHYqGhIRCTFKRGIiKS4VEsE9yc7gFZOx+fAdGwOTsfnwFr9sUmpOgIREflXqXZFICIi+1AiEBFJcSmTCMxsspktM7MSM7sj2fEkg5mtMbMFZjbXzGaG27qb2RtmtiK87xZuNzP7RXi85pvZScmN/tgzs4fMrMzMFiZsO+zjYWbXhfuvMLPrkvFZjrUDHJu7zKw0/P7MNbOLEh779/DYLDOzCxK2t7v/OzPrb2ZvmtliM1tkZl8Kt7fd7467t/sbkAasBAYDnYB5wMhkx5WE47AGyN9n2z3AHeHyHcB/h8sXAa8CBkwE3k92/DEcjzOBk4CFR3o8gO7AqvC+W7jcLdmfLaZjcxfw9f3sOzL8n8oABoX/a2nt9f8O6AOcFC7nAMvDY9BmvzupckUwAShx91XuXgc8CUxJckytxRTg0XD5UeCyhO2/9cB7QJ6Z9UlGgHFx93eA7ftsPtzjcQHwhrtvd/cdwBvA5Pijj9cBjs2BTAGedPdad18NlBD8z7XL/zt33+Tus8PlKmAJUEgb/u6kSiIoBNYnrG8It6UaB143s1lmdnO4rZe7bwqXNwO9wuVUPWaHezxS7TjdFhZvPNRc9EEKHxszKwLGAe/Thr87qZIIJHC6u58EXAh8wczOTHzQg+tVtScO6Xj8i/8DhgBjgU3AT5IbTnKZWTbwLPBld9+Z+Fhb++6kSiIoBfonrPcLt6UUdy8N78uA5wku3bc0F/mE92Xh7ql6zA73eKTMcXL3Le7e6O5NwAME3x9IwWNjZukESeBxd38u3NxmvzupkghmAMPMbJCZdQKuBKYmOaYWZWZdzCyneRk4H1hIcByaWytcB7wYLk8Frg1bPEwEKhMue9uzwz0erwHnm1m3sKjk/HBbu7NPHdFHCb4/EBybK80sw8wGAcOA6bTT/zszM+A3wBJ3/2nCQ233u5PsGviWuhHU3C8naMVwZ7LjScLnH0zQamMesKj5GAA9gL8AK4A/A93D7QbcFx6vBUBxsj9DDMfkCYIijnqC8tkbj+R4AJ8hqCAtAW5I9ueK8dg8Fn72+QQntz4J+98ZHptlwIUJ29vd/x1wOkGxz3xgbni7qC1/dzTEhIhIikuVoiERETkAJQIRkRSnRCAikuKUCEREUpwSgYhIilMikDbNzN4ys9gnBjez281siZk9Hvd7JZOZ5ZnZ55Mdh7QsJQJJWWbW8TB2/zxwnrtfHVc8rUQewWeVFKJEILEzs6Lw1/QD4fjtr5tZVvjY3l/0ZpZvZmvC5evN7IVwXPc1ZnabmX3VzOaY2Xtm1j3hLa4Jx8dfaGYTwud3CQdGmx4+Z0rC6041s78SdP7ZN9avhq+z0My+HG77FUGHvFfN7Cv77J9mZj8O959vZl8Mt58Tvu+CMI6McPsaM/t/YbwzzewkM3vNzFaa2S3hPmeb2Ttm9rIFY/n/ysw6hI9dFb7mQjP774Q4dpnZD8xsXnh8eoXbC8zsWTObEd5OC7ffFcb1lpmtMrPbw5e6GxgSxvcjM+sTxtJ8fM844i+CtF7J7qWnW/u/AUVAAzA2XH8K+HS4/BZhT0sgH1gTLl9P0NsyBygAKoFbwsd+RjDQV/PzHwiXzyQcPx/4YcJ75BH0bu0Svu4Gwl6f+8R5MkHPzy5ANkEP7HHhY2vYZy6HcPutwDNAx3C9O5BJMKrk8HDbbxPiXQPcmvA55id8xi3h9rOBGoLkk0YwPPHHgb7AunDfjsBfgcvC5zhwSbh8D/CtcPn3BIMNAgwgGBYBgrkF3iWYQyAf2Aakh3+rxDkIvsY/e6GnATnJ/j7pduxvh3NpLHI0Vrv73HB5FsEJ51De9GC89yozqwT+GG5fAJyQsN8TEIyhb2a5ZpZHMG7LpWb29XCfTIITIYRjwO/n/U4Hnnf33QBm9hxwBjDnIDGeC/zK3RvCGLab2Ynh510e7vMo8AXg5+F683g7C4DshM9YG8YOMN3dV4VxPBHGVg+85e7l4fbHCZLfC0Ad8FL43FnAeQnxjQyGxwEg14JRMwFedvdaoNbMyvjnsMmJZgAPWTDI2gsJf0NpR5QIpKXUJiw3AlnhcgP/LKLMPMhzmhLWm/jgd3ffcVKcYHyXy919WeIDZnYKsPuwIj/2Ej/Hvp+x+XPt7zMdTL27N+/TmPA6HYCJ7l6TuHOYGPb9m/zL+SBMrmcCFwOPmNlP3f23h4hF2hjVEUiyrSEokoGg+ONIfBLAzE4nGNmxkmAUxy+GI0ViZuMivM7fgMvMrLMFI7R+NNx2MG8An2uueA7rLpYBRWY2NNznGuDtw/xMEywYtbMDwef7O8GInmeFdSlpwFURXvd14IvNK2Y29hD7VxEUVTXvP5CgyOoB4EGC6SulnVEikGT7MXCrmc0hKKs+EjXh839FMEomwPcIyrznm9micP2gPJh+8BGCE+77wIPufrBiIQhOjuvC95kHfCr89X0D8LSZLSD4pf+rw/xMM4BfEkyDuJqgyGoTwVy4bxKMIjvL3V888EsAcDtQHFZkLwZuOdjO7r4NmBZWDP+IoL5iXnh8Pwnce5ifQ9oAjT4q0sqY2dkEk8R/JNmxSGrQFYGISIrTFYGISIrTFYGISIpTIhARSXFKBCIiKU6JQEQkxSkRiIikuP8PFTdX2muh8XUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pca_ft(to_arr(X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ll : 537"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transformation\n",
    "\n",
    "to_arr(X)\n",
    "- PCA\n",
    "- MinMax\n",
    "- Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformation_transformer = Pipeline(steps = [\n",
    "    ('to_arr', FunctionTransformer(to_arr)),\n",
    "    ('minmax', MinMaxScaler()),\n",
    "    ('pca', PCA())\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extraction\n",
    "\n",
    "np.array(histo_list)\n",
    "    \n",
    "- K-means\n",
    "- histo\n",
    "- models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "extraction_transformer = Pipeline(steps = [\n",
    "#     ('cluster', KMeans()),\n",
    "    \n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ColumnTransformer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = ColumnTransformer(\n",
    "    transformers=[\n",
    "        ('trans', transformation_transformer, TRANSFORMATION_LIST),\n",
    "#         ('ext', extraction_transformer, EXTRACTION_LIST)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = DecisionTreeClassifier(random_state=9)\n",
    "classifiers = [\n",
    "    ('NB', GaussianNB()),\n",
    "    ('LSVC', LinearSVC()),\n",
    "    ('SVC', SVC(random_state=9, gamma='scale')),\n",
    "    ('LOGR', LogisticRegression(random_state=9, solver='lbfgs')),\n",
    "    ('SGD', SGDClassifier()),\n",
    "    ('KNN', KNeighborsClassifier()),\n",
    "    ('CART', DecisionTreeClassifier(random_state=9)),\n",
    "    ('RF', RandomForestClassifier(n_estimators=100, random_state=9)),\n",
    "    ('ABC', AdaBoostClassifier()),\n",
    "    ('GBC', GradientBoostingClassifier()),\n",
    "    ('LDA', LinearDiscriminantAnalysis()),\n",
    "    ('MLP', MLPClassifier())\n",
    "]\n",
    "\n",
    "# classifiers = [\n",
    "#     GaussianNB(),\n",
    "#     LinearSVC(),\n",
    "#     SVC(random_state=9, gamma='scale'),\n",
    "#     LogisticRegression(random_state=9, solver='lbfgs'),\n",
    "#     SGDClassifier(),\n",
    "#     KNeighborsClassifier(),\n",
    "#     DecisionTreeClassifier(random_state=9),\n",
    "#     RandomForestClassifier(n_estimators=100, random_state=9),\n",
    "#     AdaBoostClassifier(),\n",
    "#     GradientBoostingClassifier(),\n",
    "#     LinearDiscriminantAnalysis(),\n",
    "#     MLPClassifier()\n",
    "# ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'NB': 0.7058823529411765}\n",
      "{'LSVC': 0.7647058823529411}\n",
      "{'SVC': 0.7647058823529411}\n",
      "{'LOGR': 0.7647058823529411}\n",
      "{'SGD': 0.5882352941176471}\n",
      "{'KNN': 0.47058823529411764}\n",
      "{'CART': 0.6470588235294118}\n",
      "{'RF': 0.5882352941176471}\n",
      "{'ABC': 0.29411764705882354}\n",
      "{'GBC': 0.7647058823529411}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/discriminant_analysis.py:388: UserWarning: Variables are collinear.\n",
      "  warnings.warn(\"Variables are collinear.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'LDA': 0.4117647058823529}\n",
      "{'MLP': 0.7058823529411765}\n"
     ]
    }
   ],
   "source": [
    "result_feature = []\n",
    "\n",
    "for name, classifier in classifiers:\n",
    "    pipe = Pipeline(steps=[\n",
    "#         ('preprocessor', preprocessor),\n",
    "        ('classifier', classifier)\n",
    "    ])\n",
    "    pipe.fit(X_train, y_train)\n",
    "    score = pipe.score(X_test, y_test)\n",
    "\n",
    "    result = {}\n",
    "    result[name] = score\n",
    "    print(result)\n",
    "#     print(classifier)\n",
    "#     print(\"model score: %.3f\" % score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pipeline with Cross-Validation (cross_val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n",
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/preprocessing/_function_transformer.py:141: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  return self._transform(X, func=self.func, kw_args=self.kw_args)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.59064327 0.64327485 0.59649123 0.58479532 0.63529412 0.65882353\n",
      " 0.55294118 0.58823529 0.62941176 0.64117647]\n",
      "CPU times: user 36min 43s, sys: 5min 25s, total: 42min 8s\n",
      "Wall time: 12min 46s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pipeline = Pipeline(steps=[\n",
    "    ('transform', transformation_transformer),\n",
    "    ('classifier', DecisionTreeClassifier(random_state=9))\n",
    "])\n",
    "\n",
    "scores = cross_val_score(pipeline,X_train,y_train,cv=10,scoring='accuracy')\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random search\n",
    "\n",
    "https://scikit-learn.org/stable/auto_examples/model_selection/plot_randomized_search.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # specify parameters and distributions to sample from\n",
    "# param_dist = {\"max_depth\": [3, None],\n",
    "#               \"max_features\": sp_randint(1, 11),\n",
    "#               \"min_samples_split\": sp_randint(2, 11),\n",
    "#               \"criterion\": [\"gini\", \"entropy\"]}\n",
    "\n",
    "# specify parameters and distributions to sample from\n",
    "param_dist = {\"max_depth\": [3, None],\n",
    "              \"max_features\": [1, None],\n",
    "              \"min_samples_split\": sp_randint(2, 11),\n",
    "              \"criterion\": [\"gini\", \"entropy\"]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run randomized search\n",
    "n_iter_search = 5\n",
    "random_search = RandomizedSearchCV(classifier, param_distributions=param_dist,\n",
    "                                   n_iter=n_iter_search, cv=10, iid=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/xunwei/anaconda3/envs/fyp/lib/python3.7/site-packages/IPython/core/interactiveshell.py:3319: FutureWarning: arrays to stack must be passed as a \"sequence\" type such as list or tuple. Support for non-sequence iterables such as generators is deprecated as of NumPy 1.16 and will raise an error in the future.\n",
      "  exec(code_obj, self.user_global_ns, self.user_ns)\n"
     ]
    }
   ],
   "source": [
    "X = data['LL']\n",
    "X = to_arr(X) #ll must to arr\n",
    "#sift must use after histo generated\n",
    "\n",
    "y = data['diagnosis']\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "random_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Utility function to report best scores\n",
    "def report(results, n_top=3):\n",
    "    for i in range(1, n_top + 1):\n",
    "        candidates = np.flatnonzero(results['rank_test_score'] == i)\n",
    "        for candidate in candidates:\n",
    "            print(\"Model with rank: {0}\".format(i))\n",
    "            print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n",
    "                  results['mean_test_score'][candidate],\n",
    "                  results['std_test_score'][candidate]))\n",
    "            print(\"Parameters: {0}\".format(results['params'][candidate]))\n",
    "            print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "report(random_search.cv_results_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model with rank: 1\n",
      "Mean validation score: 0.673 (std: 0.136)\n",
      "Parameters: {'max_features': None, 'max_depth': 3, 'criterion': 'gini'}\n",
      "\n",
      "Model with rank: 1\n",
      "Mean validation score: 0.673 (std: 0.136)\n",
      "Parameters: {'max_features': None, 'max_depth': None, 'criterion': 'gini'}\n",
      "\n",
      "Model with rank: 3\n",
      "Mean validation score: 0.616 (std: 0.160)\n",
      "Parameters: {'max_features': None, 'max_depth': 3, 'criterion': 'entropy'}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "report(random_search.cv_results_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = { \n",
    "    'classifier__criterion': ['gini', 'entropy'],\n",
    "    'classifier__max_depth' : np.linspace(1, 32, 32, endpoint=True, dtype='int'),\n",
    "    'classifier__min_samples_split' : np.linspace(2, 50, 49, endpoint=True, dtype='int')\n",
    "}\n",
    "\n",
    "classifier = DecisionTreeClassifier(min_samples_split=4, random_state=0)\n",
    "\n",
    "pipe = Pipeline(steps=[\n",
    "    ('classifier', classifier)\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, groups, **fit_params)\u001b[0m\n\u001b[1;32m    686\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mresults\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    687\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 688\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    689\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    690\u001b[0m         \u001b[0;31m# For multi-metric evaluation, store the best_index_, best_params_ and\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36m_run_search\u001b[0;34m(self, evaluate_candidates)\u001b[0m\n\u001b[1;32m   1147\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_search\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1148\u001b[0m         \u001b[0;34m\"\"\"Search all candidates in param_grid\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1149\u001b[0;31m         \u001b[0mevaluate_candidates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mParameterGrid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_grid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1151\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/model_selection/_search.py\u001b[0m in \u001b[0;36mevaluate_candidates\u001b[0;34m(candidate_params)\u001b[0m\n\u001b[1;32m    665\u001b[0m                                \u001b[0;32mfor\u001b[0m \u001b[0mparameters\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    666\u001b[0m                                in product(candidate_params,\n\u001b[0;32m--> 667\u001b[0;31m                                           cv.split(X, y, groups)))\n\u001b[0m\u001b[1;32m    668\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    669\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, iterable)\u001b[0m\n\u001b[1;32m   1004\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1005\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1006\u001b[0;31m             \u001b[0;32mwhile\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1007\u001b[0m                 \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1008\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[0;34m(self, iterator)\u001b[0m\n\u001b[1;32m    832\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    833\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 834\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    835\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    836\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    751\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    752\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 753\u001b[0;31m             \u001b[0mjob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    754\u001b[0m             \u001b[0;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    755\u001b[0m             \u001b[0;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[0;34m(self, func, callback)\u001b[0m\n\u001b[1;32m    199\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    200\u001b[0m         \u001b[0;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mImmediateResult\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallback\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m             \u001b[0mcallback\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/_parallel_backends.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, batch)\u001b[0m\n\u001b[1;32m    580\u001b[0m         \u001b[0;31m# Don't delay the application, to avoid keeping the input\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0;31m# arguments in memory\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 582\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    583\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    584\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 256\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/joblib/parallel.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mparallel_backend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_jobs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_n_jobs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m             return [func(*args, **kwargs)\n\u001b[0;32m--> 256\u001b[0;31m                     for func, args, kwargs in self.items]\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/model_selection/_validation.py\u001b[0m in \u001b[0;36m_fit_and_score\u001b[0;34m(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, return_estimator, error_score)\u001b[0m\n\u001b[1;32m    514\u001b[0m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    515\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 516\u001b[0;31m             \u001b[0mestimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    517\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    518\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[1;32m    354\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n\u001b[1;32m    355\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'passthrough'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 356\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    357\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    358\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/tree/tree.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    814\u001b[0m             \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    815\u001b[0m             \u001b[0mcheck_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheck_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 816\u001b[0;31m             X_idx_sorted=X_idx_sorted)\n\u001b[0m\u001b[1;32m    817\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/fyp/lib/python3.7/site-packages/sklearn/tree/tree.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, X, y, sample_weight, check_input, X_idx_sorted)\u001b[0m\n\u001b[1;32m    378\u001b[0m                                            min_impurity_split)\n\u001b[1;32m    379\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 380\u001b[0;31m         \u001b[0mbuilder\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtree_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX_idx_sorted\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    381\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    382\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mn_outputs_\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# instantiate the grid\n",
    "grid = GridSearchCV(pipe, param_grid, cv=10, scoring='accuracy')\n",
    "\n",
    "# fit the grid with data\n",
    "grid.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6060606060606061\n",
      "{'classifier__min_samples_split': 29}\n",
      "Pipeline(memory=None,\n",
      "         steps=[('classifier',\n",
      "                 DecisionTreeClassifier(class_weight=None, criterion='gini',\n",
      "                                        max_depth=None, max_features=None,\n",
      "                                        max_leaf_nodes=None,\n",
      "                                        min_impurity_decrease=0.0,\n",
      "                                        min_impurity_split=None,\n",
      "                                        min_samples_leaf=1,\n",
      "                                        min_samples_split=29,\n",
      "                                        min_weight_fraction_leaf=0.0,\n",
      "                                        presort=False, random_state=0,\n",
      "                                        splitter='best'))],\n",
      "         verbose=False)\n"
     ]
    }
   ],
   "source": [
    "# examine the best model\n",
    "\n",
    "# Single best score achieved across all params (min_samples_split)\n",
    "print(grid.best_score_)\n",
    "\n",
    "# Dictionary containing the parameters (min_samples_split) used to generate that score\n",
    "print(grid.best_params_)\n",
    "\n",
    "# Actual model object fit with those best parameters\n",
    "# Shows default parameters that we did not specify\n",
    "print(grid.best_estimator_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'grid' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-48-7a219d11d30d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mreport\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'grid' is not defined"
     ]
    }
   ],
   "source": [
    "report(grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find opt k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optk(desc):\n",
    "    if desc == 'surf':\n",
    "        name = '(SURF)'\n",
    "        method = surf_des\n",
    "    elif desc == 'sift':\n",
    "        name = '(SIFT)'\n",
    "        method = sift_des\n",
    "    else:\n",
    "        name = '(ORB)'\n",
    "        method = orb_des\n",
    "        \n",
    "    img_col = data['pixel']\n",
    "    img_col = img_col.apply(method)\n",
    "    bag = generate_bag(img_col)\n",
    "    \n",
    "    Sum_of_squared_distances = []\n",
    "    Silhouette_scores = []\n",
    "\n",
    "    K = range(2,25)\n",
    "    for k in K:\n",
    "        km = KMeans(n_clusters=k)\n",
    "        km = km.fit(bag)\n",
    "        Sum_of_squared_distances.append(km.inertia_)\n",
    "\n",
    "        labels = km.labels_\n",
    "        Silhouette_scores.append(silhouette_score(bag, labels, metric = 'euclidean'))\n",
    "        \n",
    "    with open(desc + 'sum.pkl', 'rb') as f:\n",
    "        pickle.dump(Sum_of_squared_distances, f)\n",
    "\n",
    "    with open(desc + 'sil.pkl', 'rb') as f:\n",
    "        pickle.dump(Silhouette_scores, f)\n",
    "        \n",
    "    fig, (ax1, ax2) = plt.subplots(2)\n",
    "    # fig.suptitle('Axes values are scaled individually by default')\n",
    "    ax1.plot(K, Sum_of_squared_distances, 'bx-')\n",
    "    ax1.set_title('Elbow Method For Optimal k ' + name)\n",
    "\n",
    "    ax2.plot(K, Silhouette_scores, 'bx-')\n",
    "    ax2.set_title('Silhouette Method For Optimal k ' + name)\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.savefig(desc + '.svg', dpi=300, format='svg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "desc = 'sift'\n",
    "\n",
    "if desc == 'surf':\n",
    "    name = '(SURF)'\n",
    "    method = surf_des\n",
    "elif desc == 'sift':\n",
    "    name = '(SIFT)'\n",
    "    method = sift_des\n",
    "else:\n",
    "    name = '(ORB)'\n",
    "    method = orb_des\n",
    "\n",
    "img_col = data['pixel']\n",
    "img_col = img_col.apply(method)\n",
    "bag = generate_bag(img_col)\n",
    "\n",
    "Sum_of_squared_distances = []\n",
    "Silhouette_scores = []\n",
    "\n",
    "K = range(2,25)\n",
    "for k in K:\n",
    "    km = KMeans(n_clusters=k)\n",
    "    km = km.fit(bag)\n",
    "    Sum_of_squared_distances.append(km.inertia_)\n",
    "\n",
    "    labels = km.labels_\n",
    "    Silhouette_scores.append(silhouette_score(bag, labels, metric = 'euclidean'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dd5hU5dnH8e+PqnQQLICIYo+xLmp8sUVfFexRsYuJPdHExBITNRpLXiUWYo2NCLaIECO2WLFEjQKK2GJDlCZdBQQRud8/7jPu2WF2d2bbzO7en+s618zpz5w9O/c85TyPzIwQQgih1LQodgJCCCGEXCJAhRBCKEkRoEIIIZSkCFAhhBBKUgSoEEIIJSkCVAghhJIUASog6XhJ/07Nm6QNi5mmulKXn0XSVEl71sWxSoGkxZI2qIfjVrifstb1Tf4mrfI8VltJ70pap25TWeU5z5B0ZUOdL1QuAlQzkXy5Lk2+lDLTDcVOF3z/hWaSrs1afmCy/M48j/OcpBPrJZHVn/tOScuzru/hdXj8/SS9JmmJpPmS7pHUu4D9V7k2ZtbBzKbUVRrrycnAC2Y2C0BSb0ljJM2T9KWktyUdn6yrEPwq+5tkza/M+r84GrgNOFrSmsX60MFFgGpe9k++lDLT6cVOUMrHwOCsX9ZDgA+KlJ6aGJp1fe8v9ACSWuZYdihwLzAM6A78APgG+LekrrVNdIk7FbgrNX8XMA1YD1gDOBaYXcX+q/xN0vPAZ1T8v7jHzJYBjwPH1c9HCvmKABUqM0jSlOSX6p8ltQCQ1ELSBZI+lTRH0khJnZN1IySdlbzvlfya/UUy30/SgsxxcvgceAvYO9m+G7ATMDa9kaQdJb0s6QtJb0raLVl+ObAzcEOO3OGekj5M9rlRkqr7LMn6Y5N18yWdX9MLKWmzJAfzhaR3JB2QWnenpJslPSZpCbB71r4CrgYuM7N7zWypmX0OnAgsBn6dbHe8pJck3ZDkLP4raY+qro1SxZ9JOm6S9HiyzUuS1pY0TNLC5HjbpNJ1nqSPJS2SF8EdXMNrc4g8d79FjnV9gA2AV1OL+wN3mtkSM1thZm+Y2eM1OXc1ngP2rYfjhgJEgAqVORgoA7YFDgR+liw/Ppl2x788OgCZYPA8sFvyfldgCrBLav5FM1tZxTlHUv6r9QjgITynAHjQAx4FLgO6AWcDYyT1MLPzgReB03PkDvfDv9i2BAaTBMGqPoukzYGb8V/oPfFf63kXqaXS3Bp4GHgSWBM4A7hH0iapzY4CLgc6Atl1N5sAfYAH0guT6zgG+N/U4h3wnGh34CLgH5K6VXNt0gYDFyT7fwO8AryezI8Grklt+zEe9DoDfwTuVoH1RJJ+ClwJ7Glmb+fY5IfAFDNbkVr2H+BGSUckAay+vAdsVY/HD3ko2QAlaXjyqzbXjZu97S6SXpe0IikOSa8bkvx6/lDSkPpLcaPwz+RXfGY6qYptrzSzBWb2GV60dGSy/GjgGjObYmaLgd8BRyRFc88DA5Jc0i7AUOB/kv12TdZX5UFgtyQXcxwesNKOAR4zs8fMbKWZPQVMAAZVc9wrzOyL5LOMA7bO47McCjxiZi+Y2TfAhUBVwRXg7NS1nZcs2xEPfFeY2XIzexZ4hPLrCfCQmb2UfKZlWcfsnrzOynG+Wan1AHOAYWb2bVK8+D6F5QIeNLOJSRoeBJaZ2Ugz+w64H/g+B2VmD5jZzCTN9wMfAtsXcK4zgXOA3czso0q26QIsylp2GB5sLwQ+kTRJUv8qzpPrb5KPRXjwDUVUsgEKuBPYJ89tP8N/Cd+bXpgUE12E/7LcHrioGZTZV+UgM+uSmm6rYttpqfef4rkIktdPs9a1AtYys4+BJXgA2Bn/Ip6Z5BaqDVBmthTPIV0ArGFmL2Vtsh5wWDrIAgOA6n65f556/zUeMKr8LMm676+BmS0B5ldznqtS1zYTOHoC07Jyjp8CvVLz6WudLfOlmuszrpNaDzDDKvb+nP675SNdl7M0x3zmuiHpuCQ4ZP4OW1AxWFbnHOBGM5texTYL8Vzl98xsoZmdZ2Y/wP9Ok/AfXqrkGLn+JvnoCHxZwPahHpRsgDKzF4AF6WVJPca/JE2U9KKkTZNtp5rZZFb9hbs38FSSE1gIPEX+Qa+5Wzf1vg8wM3k/Ew8U6XUrKP8yex7PfbQxsxnJ/BCgK/5lUp2RwFnA3TnWTQPuygqy7c3simR9oV3zV/VZZpG6BpLa4cV8hZoJrJtV99YHmJGaryrd7wPT8ZzD95LjHQI8k1rcK+uLOv13q7NhCySth7d0Ox3/IdEFeBuoLEjkshdwgaRDqthmMrC+KmmSbmbzgKvwINytgHPnYzPgzTo+ZihQyQaoStwKnGFm2+H1DzdVs30vKv46nU7FX66hcudI6ippXeBXeBEPwH3AryWtL6kD8Cfg/lQ9wfP4F9cLyfxzyfy/k6Ki6jyP16tcn2Pd3cD+kvaW1FLSapJ2U3lz69l4XVK+qvoso4H9JA2Q1Aa4hJr9v7yK59rOldRa3qhjf+Dv+eyc5IjOxr/Mj0o+89rA7UAnIN00f03gl8l5DsO/ZB9L1hV6barSHg94c+H7uqRVGjlU4x38x+KN6UYjaUnu6iNSRYeSrpS0haRWkjoCpwEfmVl1udtC7Yq35AtF1GgCVPIFshPwgKRJwC1UX7QTKnpYFZ8BebCKbR8CJuK5nkeBO5Llw/Gmvi8AnwDL8Ir/jOfx4pFMgPo30C41XyVzz5jZghzrpuENNn6PfzlOw4uKMvfxX4BDk1Zn1+Vxuko/i5m9A/wCLzaehRc3VVUcVdnnWY4HpIF4cdxNwHFm9t8CjnE/3ljj13gx47vA6sD/ZH0xvwpslJzncuDQ1PpCr01V6XkXb1n4Ch74fghkF8fmc5w38QYst0kaWMlmt+CfPaMdXj/2Bd4IZz0gZ4CrKUmr4fWaI+ryuKFwKuUBCyX1xSuqt5DUCXjfzCoNSvIHOh8xs9HJ/JF4JewpyfwtwHNmdl99pz2EhiR/WPVEMxtQ7LTUJUltgTeAPSx5WLcBznkGsK6ZndsQ5wuVazQ5KDP7Cm+1cxj48yGSqmsG+gSwV1JU1RUv936inpMaQqgjZvaNmW3eUMEpOef1EZxKQ8kGKEn34UUIm0iaLukEvFnwCZLexMuwD0y27S8pU5F8i6R3AJJiokuB8cl0Sa6ioxBCCKWnpIv4QgghNF8lm4MKIYTQvOXV5X1D6t69u/Xt27fYyQghhFAPJk6cOM/MeuSzbckFqL59+zJhwoRiJyOEEEI9kPRp9Vu5JlXEN3QojBtXcdm4cb48hBBC49KkAlT//jB4MNyXPOU0bpzP96+qK8kQQgglqeSK+Gpj993h/PPhqKNgzBh4/nkYNcqXhxBCaFyaVA4K4Oc/h549PUAdeWQEpxBCaKyaXIB66SVYtgzatoWbb4Ynnyx2ikIIIdREkwpQmTqn0aPhnntgxQo46KBVG06EEEIofU0qQI0fX17ndMghcNppsHQp3HVXsVMWQgihUCXX1VFZWZnV1XNQy5bBDjvAzJkwaRL0ipGgQgihqCRNNLOyfLZtUjmobKutBvffD19/DcccA9/lM1xeCCGEktCkAxTAppvCTTfBc8/BZZcVOzUhhBDy1eQDFMCQIXDssXDJJf5sVAghhNLXLAIUeC5qww39Id65c4udmhBCCNVpNgGqQwevj5o/H44/HlauLHaKQgghVKXZBCiArbeGq6+Gxx6Da68tdmpCCCFUpVkFKPCukA4+GM47D157rdipCSGEUJlmF6AkuOMOfybq8MPhiy+KnaIQQgi5NLsABdC1qw/JMW0anHQSlNizyiGEEGimAQrgRz+Cyy/3fvtuuaXYqQkhhJCt2QYogHPOgb33hjPPhMmTi52aEEIIac06QLVoASNHQrduXh+1ZEmxUxRCCCGjWQcogDXXhLvvhvffh9NPL3ZqQgghZDT7AAXw4x/7dOedHqwyxo2DoUOLlqwQQmjWIkAlzjsPWrXyVn0ffFA++GH//sVOWQghNE+tip2AUrHnnj4K7xFHwE47edPz0aN98MMQQggNL3JQKYMHe6/n8+fD8uXeeCKEEEJx1CpASRouaY6kt6vZrr+kFZIOrc356tu4cd5P3ymn+CCHP/oRPPVUsVMVQgjNU21zUHcC+1S1gaSWwJXAk7U8V73K1DmNGgV//av3NLF8OeyzjzeeCCGE0LBqFaDM7AVgQTWbnQGMAebU5lz1bfx4D06ZOqfBg+HBB2GDDeCnP/XBDqNLpBBCaDj1WgclqRdwMHBzfZ6nLpx77qoNIvbfH955B447Di66yFv4ffttcdIXQgjNTX234hsG/NbMVkqqdCNJJwMnA/Tp06eek1SYNm28iG+99eDSS2H6dHjgAejYsdgpCyGEpq2+W/GVAX+XNBU4FLhJ0kHZG5nZrWZWZmZlPXr0qOckFU7yIr7bboOnn4Zdd4VZs4qdqhBCaNrqNUCZ2fpm1tfM+gKjgZ+b2T/r85z16cQT4eGH/UHeHXeEd98tdopCCKHpqm0z8/uAV4BNJE2XdIKkUyWdWjfJKz0DB8ILL3gLv512gueeK3aKQgihaapVHZSZHVnAtsfX5lylZNtt4ZVXYNAgH65jxAjvgSKEEELdiZ4kaqhvX3jpJS/qO/JID1bPPltxm+hsNoQQai4CVC107QpPPum5p8cfh3339UYUEJ3NhhBCbUWAqqW2bb2T2XPPhWXLPEj99rflvVJEZ7MhhFAzEaDqQIsWcOWVcMMN3nhi6FDYeWfYbbdipyyEEBqvCFB1aPPNoUsXWGcd7yZp663h7Sq70Q0hhFCZCFB1JFPn9I9/wLRp8JvfwFtvwVZbwZlnwpdfFjuFIYTQuESAqiPpzmZbtoSrr/Zg1b8/XHcdbLIJjBwZHc6GEEK+IkDVkVydzR50EPznPx68+vaFIUO8bmrSpKIkMYQQGpUIUA1gu+3g5Zdh+HDvJmm77eD002HhwmKnLIQQSlcEqAbSooWPK/X++/Dzn8PNN8PGG8Mdd8DKlcVOXQghlJ4IUA2sa1e4/np4/XXYdFPvgLZvXx/FNy16oQghNHcRoIpkq62809mRI2HxYjjtNB8gcf786IUihBCg/gcsDFWQ4Nhj4YADPCc1erQ/Q9WypT/0G71QhBCas8hBlYDOnX2U3lNP9SHlv/3WA9aAAd6N0jffFDuFIYTQ8CJAlYhx4zwHdeGF3hvFqafC55/DMcdA797ev9+UKcVOZQghNJwIUCUgU+c0apQPLf/AAx6sbrkFnnjCc1JXXQUbbujDejz8MHz3XbFTHUII9SsCVAlI90IB/jpqFEycCHvt5f36ffqp564mTfI6qw02gMsv91zW0KEe5NKiFWAIobGTlVjfO2VlZTZhwoRiJ6NkffstPPSQP0f17LPQqpXnsN54w7tW+vGPK+bIoqFFCKGUSJpoZmV5bRsBqvF6/31/furOO+GLL7z13y67eC5rzJgITiGE0lNIgIoivkZsk03g2mthxgzvkWLttT33tHChd6V04YWesyqx3yAhhJCXCFBNQLt2sP763hz9V7+C9u2hdWv4059g2229vuqss+Cll6JbpRBC4xEBqglI1zkNG+at/GbM8JaAt9/uAylef73XVfXq5b1WPPWU12dFA4sQQqmKANUEVNYK8MMP4YQT4NFHYe5cuPdeD1IjR3rrwLXWgmee8WFB/vUv3ze6WQohlIpoJNEMLV0KTz7prf4efrh82I9NNoHp0+GaazywtWxZ3HSGEJqeaMUX8vbtt/D883Deef7cVUanTp7b2nlnbxlYVgZt2hQvnSGEpqGQABWdxTZzrVt7TinzIPCNN3o3S/Pne2/rjz3m2622Guy4Y3nA2nFH6NDB66r696/YpH3cOC92PPfc4nymEELTEHVQzVx2N0ujR8Ott8Lhh8O778KcOV4UeNppsGiR917xv//r/QXusANMmOB1WA89VPF4UYcVQqityEE1c5U1sBg/3t/36AEHH+wTwFdfwSuvwIsveg5r7Fhv3n7QQdCtm49tdeyxsGwZzJrlz2ZJxft8IYTGK+qgQq0sW+bB7KKLPPfUpYv3apGx5pqw9dY+bbONv260kRcrRvFgCM1P1EGFBrPaarBiBbz1ltdh3Xyz56q6dPEulyZN8t4shg2D5ct9n9VXhy239NzZpZfClVfC8cfDq6+WFzeGEELkoEKtZHdMW1lHtcuXw3//Wx60MoErndtq2dJbDu65pwewrbaCPn2iiDCEpiSamYcGU5tiOjP47DPfbtQo2HRTb/b+8cfl23TqVB6sttzSpx/+0LtziiLCEBqfCFCh0cjkuE47zYsHR43yoPP22/DmmzB5cvnrokW+jwT9+sE66/izW+edB0ccAVOnwlFHxTAjIZSyCFChUci3eBA8tzV1qgeqdND68MOK2/Xu7Q0xNtkENt64fFpnnYpFhZH7CqE4opFEaBSqa+KeJnmP7euvDwceWL588WLvwX34cPif//Fm7R98AE8/7S0MMzp0qBiwVqyAn/zE+yXcf/+KwTGEUBoiBxUatVxFhLvv7sOKTJ/uweqDD3xwx8z7qVMrDjvSrp034thtN9hpJy8+3GADf63sOa7IgYVQM5GDCs1CdpHg7rtXnO/Tx6c996y43zffwJQpHrRuuMF7dF9vPQ9ezzxTcYDH1Vf3YJUJWJnXtdaqvHgyhFA3ahWgJA0H9gPmmNkWOdYfDfwWELAIOM3M3qzNOUPIKKSIMK1tW9hsM/j8c6/Lyjy/NWqUFxNOneoB7OOPK74+8wx8/XXFY+25p9d7zZ7tAWrqVHjuOejb15e3quQ/LHJgIVSvVkV8knYBFgMjKwlQOwHvmdlCSQOBi81sh6qOGUV8oSEU0kAjw8z7JkwHrjFj/CHljh29Piz979SypQepvn297qxv3/Jp5kz45S8LO38ITUGDtuKT1Bd4JFeAytquK/C2mfWqarsIUKEh1EUOJrv+6557vPhv6tSK0yef+OvMmRUDWIukq+Z11/Xc3JFHenoywaxnz9xjckXuKzRmpRqgzgY2NbMTc6w7GTgZoE+fPtt9+umntUpTCPWtJjmwb76BadMqBq+HHvJnvjp08BxYWqtWXoeWnftauBAuvtjPtccekfsKjUvJBShJuwM3AQPMbH5Vx4scVGgM6iMHdvfdHoxy5b6mTvVcVrauXT2w7b67D3+SaRjSp4/nzNq3r5+0h1BTJdWKT9KWwO3AwOqCUwiNRa4v8kxLwnxU1QJxr71y77N0qXcNlQlaI0f60Ce9e/vYXU8/XbH5PPgQKOmg1aePB7Sf/ARuucVfX3wx/xaIEdxCQ6rXACWpD/AP4Fgz+6A+zxVCY1KTFoirr+49ZGyyiQeFDz+s2AJxwACv5/rsMy9K/Oyz8umTT+D55+HLL8uPd/jh/tqiBWy4ofc4P2qU13316lXxtVs3fx6sf/9oXh8aTm1b8d0H7AZ0B2YDFwGtAczsr5JuBw4BMpVKK6rL2kURXwhVq0n9V8aXX5YHrxtvhMce866h1l7bg9uMGTA/RzlH27YeqHr29Lqx117zh5pfe83rwwYN8nUdO1bd+3zkwEL0xRdCE1Yf9V/p4LZsmdd3zZjhQSsTuNKvn3ziPc9na9/eA9U665QHtPT8Z5/Bb34DDzxQeHCN4NY0RIAKIVSqNjmw9P4/+xncdhtccEF5DiwzzZpV/j774eaMrl29h/odd4QttvDeOdZc01/T7zt18lxZbdMdAa40lFQjiRBCaalpDxywalDYZ5/y+aOOWnV7M/jqq4oBa+ZMf8D5tdfKm82PHu1Fi7l+L7dtWx6w+vXz4sRtt/VeQM44w5vvT5jgIzT36OF9K+YS9WeNT+SgQgh5q8/ixRUrYN487zZqzhx/zUzp+Y8+Kh8bLJd27cqDVY8e0L17+fv58+HWW+GQQ+Cf/4QRI2C//aoftTlyX3UnivhCCCWprooXTz3Vg9uwYZ6rmju3fJo3r+J8ZqqsqLFVKw9imUCW63XGDLj8cm+af+CB8NJLUbxYU1HEF0IoSXVZvPjjH5fPH3BA9ef++msYO9ZzbgMHwiOPwDHHeF1YOrhNnuyvCxasWuQ4eHD5+zXXhLPO8ib4a6xR8TV72YYb1rx4sTkHt8hBhRAahdp+UReae1uxwuvH0sFr+HBvmr/DDv482oIFXmyYfs1+WDpN8kYfS5bAllt67i8T0Lp182CZ/f6tt+C445pO45Ao4gshhCx1FeByNc3PWLnSG4UsWJA7eD36qDcO2XBDLz5csMCD4IIFHhAr06oVfPed58a++ALKymCjjcoDWWbKnu/aFV5+ubRaP0aACiGEOlTburP0MXIFODPvgioTrNKBKzM99RS88Ya3fOzSxdcvXOgBsSrt2vm0cKE/izZ7Nuy8s+cAu3SBzp19Sr9Pz7/6qvc6UldDw0SACiGEOtTQxYuV7Z8ruK1Y4bmqTMDKTJkgl5leeslHkV57bQ9YX3zhPYt8913V527VyrdfvNiLGh95pHY950eACiGEElKbAFefuTczbzzy5ZflASv7fWb+hRd8aJgLL4RLLqn5tYgAFUIITUSxc2/pY1RV/5avCFAhhBCA0ghwaRGgQggh1IloxZciaS7lw3M0R92BecVORCMU161m4roVLq5ZzWSu23pm1iOfHUouQDV3kibk++silIvrVjNx3QoX16xmanLdWtRXYkIIIYTaiAAVQgihJEWAKj23FjsBjVRct5qJ61a4uGY1U/B1izqoEEIIJSlyUCGEEEpSBKgQQgglKQJUCZE0VdJbkiZJiqeVKyFpuKQ5kt5OLesm6SlJHyavXYuZxlJTyTW7WNKM5H6bJGlQMdNYiiStK2mcpHclvSPpV8nyuN8qUcU1K/h+izqoEiJpKlBmZvEQYBUk7QIsBkaa2RbJsqHAAjO7QtJ5QFcz+20x01lKKrlmFwOLzeyqYqatlElaB1jHzF6X1BGYCBwEHE/cbzlVcc0GU+D9Fjmo0OiY2QvAgqzFBwIjkvcj8H+IkKjkmoVqmNksM3s9eb8IeA/oRdxvlarimhUsAlRpMeBJSRMlnVzsxDQya5nZrOT958BaxUxMI3K6pMlJEWAUU1VBUl9gG+BV4n7LS9Y1gwLvtwhQpWWAmW0LDAR+kRTLhAKZl1tH2XX1bgb6AVsDs4Cri5uc0iWpAzAGONPMKoxhG/dbbjmuWcH3WwSoEmJmM5LXOcCDwPbFTVGjMjsp+86Ugc8pcnpKnpnNNrPvzGwlcBtxv+UkqTX+RXuPmf0jWRz3WxVyXbOa3G8RoEqEpPZJhSKS2gN7AW9XvVdIGQsMSd4PAR4qYloahcwXbOJg4n5bhSQBdwDvmdk1qVVxv1WismtWk/stWvGVCEkb4LkmgFbAvWZ2eRGTVLIk3QfshnffPxu4CPgnMArogw/XMtjMolFAopJrthte3GLAVOCUVL1KACQNAF4E3gJWJot/j9epxP2WQxXX7EgKvN8iQIUQQihJUcQXQgihJEWACiGEUJIiQIUQQihJEaBCCCGUpAhQIYQQSlIEqBBCCCUpAlQIIYSSFAEqhBBCSYoAFUIIoSRFgAohhFCSIkCFEEIoSRGgQgghlKQIUE2MpKMlPZmaN0kbJu/vlHRZ8VJX+uryGkm6WNLddXGsUiDpr5IurKdjf3+f5lj3nKQTCzjW/0k6s+5SV+351pL0nqS2DXXO5iICVCMkaYCklyV9KWmBpJck9Qcws3vMbK9ipzEt1xdMVV9IeR7TJM2R1Cq1rHWyLK8u+iUdL+nfNU1DbUjaTdJKSYtT08N1ePzeku6RNF/SEkmvSdqvgP1XuTZmdqqZXVpXaawPknoAxwG3pJb9XtInyTWeLun+1Lrv783K/iZJYM7ML5f0bWr+cTObDYwDTm7oz9vURYBqZCR1Ah4Brge6Ab2APwLfFDNdRbIQGJiaH5gsayxmmlmH1LR/oQdIB+jUsm7Av4HlwA/wMaCuBe6VdGhtE13ijgceM7OlAJKGAMcCe5pZB6AMeKaK/Vf5mySBuUOy/5+A+1PrM/ffPcAp9fapmqkIUI3PxgBmdl8yfPJSM3vSzCZDXrmCrpIelbRI0quS+mVWSNpJ0vgkZzZe0k6pdVMl7Zmar1B8JWnHJFf3haQ3Je2WLL8c2Bm4IfnFeYOkF5Ld3kyWHZ5su5+kSckxXpa0ZTXX4i7813LGccDI9AaSOku6Q9IsSTMkXSappaTNgL8CP0rS8EUdXKP1JT2f7PcUHhgKJqmtpGGSZibTsEzxUfIrf7qk30r6HPhbjkP8GlgMnGBmnyf3yH3A5cDVkpQcyyT9UtIUSfMk/VlSi8qujVLFn6l0nJvkWmdJOkjSIEkfyHP2v099pu0lvZL8bWcl90GbGlybdSRNlnROJZsMBJ5PzfcHnjCzjwGS63FroefNw6vABpLWq4djN1sRoBqfD4DvJI2QNFBS1wL3PwLPcXUFPsK/tDK/uh8FrgPWAK4BHpW0RnUHlNQr2fcyPFd3NjBGUg8zOx8fXfP05Bfn6Wa2S7LrVsmy+yVtAwzHf4WugRfRjFXV5fr/BHaR1CW5Djuz6tDbdwIrgA2BbYC9gBPN7D3gVOCVJA1d6uAa3QtMxAPTpZQPCV6o84Ed8dFHtwK2By5IrV8bv87rkbtY6X+BMWa2Mmt5ZgTYjVPLDsZzFdsCBwI/q+bapK0NrIbn4v8A3AYcA2yH/y0ulLR+su13eODsDvwI2AP4eeWXYFXJsZ4HbjCzP1ey2Q+B91Pz/wGOk3SOpDJJLQs5Z77MbAV+r2xVH8dvriJANTJm9hUwAB82+TZgrqSxktbK8xAPmtlryT/UPfiXIMC+wIdmdpeZrUh+cf8XyKfY6Ri8WOUxM1tpZk8BE4BBBXy0k4FbzOzVJGc4Ai+23LGKfZYBDwOHJ9PYZBnglddJGs40syVmNgcv6jqimrQUfI0k9cF/rV9oZt+Y2QtJ2qrSM8lRZKbByfKjgUvMbI6ZzcWD5bGp/VYCFyXnWZrjuN2BXENpz0qtz7jSzBaY2WfAMHxY7nx9C1xuZt8Cf0+O+xczW2Rm7wDvknxhm9lEM/tPct2m4j9Adi3gXJvj9TwXVdp8PaQAABmASURBVJMD6gIsysyY2d3AGcDeeHCbI+m3Vexf2d8kH4uS84c6skr5dSh9yS/c4wEkbQrcTf5fLp+n3n8NdEje9wQ+zdr2U/zXcXXWAw6TlA5mrfEvlHytBwyRdEZqWZskXVUZCfwfICD7i2e9JB2zklIt8B9l06o5Zk2uUU9goZktyVq3bhXnmWlmvXMszz7Pp1S8DnPNbBmVmwesk2P5Oqn1GelrkX2e6sw3s++S95lAOTu1finJtZO0MZ7jLAPa4d89Ews419F4DmV0NdstBDqmF5jZPcA9kloDByXvJ5nZEzn2r+xvko+OwBfVbhXyFjmoRs7M/osXY21Ry0PNxL/Q0/oAM5L3S/Avloy1U++nAXeZWZfU1N7MrsgkM4/zT8N/jaeP0S7JpVTlRfyLdy28YUD2Mb8BuqeO2cnMflBAutKqukaz8Lqr9lnraiL7PH2SZRnVpftp4CeSsv+/B+PX5IPUsnQATZ+n0GtTnZvx3OZGZtYJ+D3+oyJfF+OB9d5qiukmU7EI83tm9q2ZPZBsU9v/lwrkjVU2BN6sy+M2dxGgGhlJm0o6S1LvZH5dPOf0n1oe+jFgY0lHSWqVNFzYHG8xCDAJOELelLsMSLcGuxsv5to7aYCwWlKJnvklOhvYIOt82ctuA06VtINce0n7SupIFczM8GLIA5L36XWzgCfxhgGdkgYA/SRlipZmA70LqKyv9BqZ2ad4seYfJbWRNID8ikdzuQ+4QFIPSd3x+p1Cnqe6FugM3CFp7eTvcSRet3VO1nU6R1LX5D76FZBpgl3otalOR+ArYHGS6z+twP2/BQ4D2gMjcwTfjMdIFR3KGw3tK6lj8vcfiLdsfLXgT1C17YGpyX0Q6kgEqMZnEbAD8KqkJXhgehs4qzYHNbP5wH7JceYD5wL7mVmmOOhCoB9ehPJHvEFAZt9peAX774G5+K/0cyi/v/4CHCppoaTrkmUXAyMy5fxmNgE4CbghOcdHJMWYeaT9naTOI5fj8KLCd5Pjjqa8qOtZ4B3gc0nzcu9e4TzVXaOj8L/NAuAisloUFuAyPNhNBt4CXk+W5SVJ5wC8AcO7SVp/AxxrZvdnbf4QXtQ2CW8AckeyvKBrk4ez8euzCP8xkp2OapnZcuAneG55eCVBaiQwSNLqyfxX+H35GV78NhQ4zczq+vm3o/GWj6EOKetHZwihmZA/0LyRmX1U7LTUJUl/AuaY2bAGOt+aeAOMbaqpGwwFigAVQjPVVANUaDqiiC+EEEJJihxUCCGEkpRXDkrSPpLel/SRpPNyrN9F0uuSVijV15ekrZPuTd5Juic5vC4TH0IIoemqNgeVPHPwAd59ynRgPHCkmb2b2qYv0AlvqTPWzEYnyzfGWwJ/KKkn3lpoMzOr9GG27t27W9++fWvxkUIIIZSqiRMnzjOzHvlsm09PEtsDH5nZFABJf8ebFH8foJKuS5BUoe8vM/sg9X6mpDlAD6p42rpv375MmDAhn7SvYuhQ6N8fdt+9fNm4cTB+PJx7bo0OGUIIoQ5JyvtZsXyK+HpRsTuU6eTX/U12orbHn0f5OMe6kyVNkDRh7ty5hR76e/37w+DBHpTAXwcP9uUhhBAalwZpxSdpHXxohJ/m6GEZM7vVzMrMrKxHj7xyfjntvjuMGAH77QfnnefBadSoijmqEEIIjUM+AWoGFfvr6k15/2zVkg+w9yhwvpnVtjueanXpAl9/DVdeCaedFsEphBAaq3wC1HhgI/lgbG3woQrG5nPwZPsHgZGZhhP17ZtvYLXV/P1115UX94UQQmhcqg1QyZg4pwNPAO8Bo8zsHUmXSDoAQFJ/SdPxzhxvkZTpF20wsAtwvHyk1EmSts5xmjqRqXMaMwbWXx86dKhYJxVCCKHxKLkHdcvKyqwuWvE98wzsuacHqO22i1Z8IYRQCiRNNLOyfLZtUl0dnXtueZ3THnvAySfD6NGwayHjdoYQQigJTSpAZRs6FHr2hJ/9zOumQgghNB5NOkB17gy33grvvguXXlrs1IQQQihEkw5QAAMHwpAhcMUV8MYbxU5NCCGEfDX5AAVwzTXQowf89Kfw7bfFTk0IIYR8NIsA1a0b/PWv8OabnpMKIYRQ+ppFgAI48EA44givi3r77WKnJoQQQnWaTYACuP567wrpZz+DFSuKnZoQQghVaVYBqnt3uOEGH37jmmuKnZoQQghVaVYBCuCww+Dgg+EPf4D33y92akIIIVSm2QUoCW66Cdq186K+774rdopCCCHk0uwCFMDaa8Nf/gIvv+xFfiGEEEpPswxQAMccA4MGwe9+Bx+vMsZvCCGEYmu2AUqCW26B1q3hpJNg5Srj/IYQQiimZhugAHr3hquv9vGibr212KkJIYSQlleAkrSPpPclfSTpvBzrd5H0uqQVkg7NWjdE0ofJNKSuEl5XTjjBx4065xz47LNipyaEEEJGtQFKUkvgRmAgsDlwpKTNszb7DDgeuDdr327ARcAOwPbARZK61j7ZdUeC224DMx8/qsTGbwwhhGYrnxzU9sBHZjbFzJYDfwcOTG9gZlPNbDKQXZOzN/CUmS0ws4XAU8A+dZDuOtW3r+einngC7ryzfPm4cT6mVAghhIaXT4DqBUxLzU9PluUjr30lnSxpgqQJc+fOzfPQdeuMM7zBxBlnwMyZHpwGD/Yh5EMIITS8kmgkYWa3mlmZmZX16NGjKGnYYw8YPhyWLIHtt4dDDoFRo8qHkA8hhNCw8glQM4B1U/O9k2X5qM2+De6YY7wbpBkzYOFCuPlmH403hBBCw8snQI0HNpK0vqQ2wBHA2DyP/wSwl6SuSeOIvZJlJWncOHjxRTj7bO8K6eGHYYstPHB9+GGxUxdCCM1LtQHKzFYAp+OB5T1glJm9I+kSSQcASOovaTpwGHCLpHeSfRcAl+JBbjxwSbKs5GTqnEaNgj//GR55BNq392UPPgibbeZ9902dWuyUhhBC8yArsXbVZWVlNmHChAY/79Ch3iAiXec0bpwPzTFkiI/Ee/PN3rnsiSfC+ef7g74hhBDyJ2mimZXltW0EqPzNmAF/+pM/N9WiBZxyivflt/baxU5ZCCE0DoUEqJJoxddY9OoFN97o9VHHHuvvN9jAe6H4wx88x5UWz1GFEELNRYCqgfXW81zU++/7AIjXXOP1VvvuCw895NvEc1QhhFA7EaBqoV8/GDEC3nkHDjoIli71ZuqHHVbe4CKeowohhJqJAFUHNt0U7rsPJk+Gnj1h9GjYaacITiGEUBsRoOrQvHmwbBmsvz6MHQu//GWxUxRCCI1XBKg6kqlzeuAB731ip53g+uu9SXoIIYTCRYCqI+PHl9c5rbYaPPec9+93xx3eFL3EWvOHEELJa1XsBDQV555bcb51a3jySfj5z/0h30WL4Lrr/PmpEEII1YsAVY9atPDeJzp2hKuu8iB1xx3QKq56CCFUK74q65nkD+t26uQP8y5eDPfeC23bFjtlIYRQ2qLAqQFIcOGFcO218I9/wIEHwtdfFztVIYRQ2iJANaAzz4Tbb/e6qX32ga++KnaKQgihdEWAamAnnOAP9b7yirfymz+/2CkKIYTSFAGqCA4/3MeYeust2HVXmDWr2CkKIYTSk1eAkrSPpPclfSTpvBzr20q6P1n/qqS+yfLWkkZIekvSe5J+V7fJb7z22w8ef9wHQNx55xgIMYQQslUboCS1BG4EBgKbA0dK2jxrsxOAhWa2IXAtcGWy/DCgrZn9ENgOOCUTvII/1Pv00z7OVP/+3jt6RgzVEUJo7vLJQW0PfGRmU8xsOfB34MCsbQ4ERiTvRwN7SBJgQHtJrYDVgeVANA1I2XFH7xJp/nzYYQfvkeLZZ2OojhBCyCdA9QKmpeanJ8tybmNmK4AvgTXwYLUEmAV8BlxlZguyTyDpZEkTJE2YO3duwR+isTvxRPjb3/xB3u23hz33hLXW8sYUw4Z5q7/p0yvvLmno0BgsMYTQ9NT3g7rbA98BPYGuwIuSnjazKemNzOxW4FbwId/rOU0lacgQmDTJA9K220L79v7M1G23lW/TqRNsvnnF6Qc/gLKyiuNPZTquHTWqeJ8nhBBqK58ANQNYNzXfO1mWa5vpSXFeZ2A+cBTwLzP7Fpgj6SWgDJhCqGDcOLj7bn+g9+abPbjsthvMneu9o6enRx+F4cPL9+3QwYejHzjQiwUnTYLf/AZatvR6rbXWgs6d/YHhbEOH+j7psavGjfOixuz+BUMIoSHlE6DGAxtJWh8PREfggSdtLDAEeAU4FHjWzEzSZ8CPgbsktQd2BIbVVeKbinSOZ/fdfUrPr7mmB6u0+fNXDVwzZ8K//+3rL7nEp4w2bTxQZU9ffQWXXQYXXwyHHAJTpkTuK4RQGmR5jAMhaRAeWFoCw83sckmXABPMbKyk1YC7gG2ABcARZjZFUgfgb3jrPwF/M7M/V3WusrIymzBhQq0+VGNTF7mYTJA75RT461/hyithvfVg9uzKpzlzYMWKisdp1QpOP91zct261d1nDCEEAEkTzawsr23zCVANqTkGqNrKzoFlz1dm5UpYuNCD1RVXwF13QY8eXqzYpg3sv7/Xje2zjw8fEkIItVVIgIqeJJqA9GCJ4K+jRvnyqrRoAWus4QHq8cc912QGt9wCp50GL7wABxzg9Vtnngmvvx4DL4YQGk7koJq5qnJfAwbAE0/AiBEwdiwsX+6tBocMgaOPhp49o5FFCKEwkYMKeasq99W6tXfJ9MAD8PnnXrfVubMHnnXX9aK/L7/0gJZ5DisT4OIh4xBCbUUOKhTsww+9vmrkSPj0U1h9da/POvFEuP/+6uu+QgjNV+SgQr3aaCNvwj5lCjz3HBx5pAeoG2+Egw6K4BRCqBsRoEKNtWjhw4Ucc4w/LNy5M9xxh/eGEUIItRUBKtRKps5pzBh4+23o3Rt+/Wv4v/8rdspCCI1dBKhQK+lGFr17w8SJXgR4wQXRG0UIoXbqu7PY0MRlNyXv0cOD1v77wxFHeFdKJ55YnLSFEBq3yEGFOte5M/zrX7D33nDSSXD11cVOUQihMYoAFepFu3bw0ENw2GFw9tnwhz9ELxQhhMJEEV+oN23a+KCLnTrBpZfCF194C78W8bMohJCHCFChXrVs6YMuduoE117rdVK33+69pocQQlXiayLUO8nrobp0gYsu8iB1333Qtm2xUxZCKGVR2BIahOT1UMOGwYMPeiu/JUuKnaoQQinLK0BJ2kfS+5I+knRejvVtJd2frH9VUt/Uui0lvSLpHUlvJYMbhmbqV7/y4eqfeQb22svrpUIIIZdqA5SklsCNwEB8ZNwjJW2etdkJwEIz2xC4Frgy2bcVcDdwqpn9ANgN+LbOUh8apZ/+1B/i/c9/oKzMx6PKGDfOh/AIIYR8clDbAx+Z2RQzWw78HTgwa5sDgRHJ+9HAHpIE7AVMNrM3Acxsvpl9VzdJD43ZIYd4d0gff+xB6rPPYqiOEEJF+QSoXsC01Pz0ZFnObcxsBfAlsAawMWCSnpD0uqScQ9hJOlnSBEkT5s6dW+hnCI3UuefCddfBjBmw2WYwcKCP4Lt4sQ/jEc9NhdC81XcjiVbAAODo5PVgSXtkb2Rmt5pZmZmV9ejRo56TFErJGWd4V0hff+3jSg0f7kGqb19v9bfzzvCLX/gw9K+8AosWle87dGj5QIkZ+RYR1mbfutg/hFC9fALUDGDd1HzvZFnObZJ6p87AfDy39YKZzTOzr4HHgG1rm+jQdIwb5636LrzQn4165BF4+WUfvfeYY3ybu++GU0+FnXby56n69YODD/be0w86yAdP/Pprb3iRbxFh//61Gwm4tvuHEKpX7Yi6ScD5ANgDD0TjgaPM7J3UNr8Afmhmp0o6AviJmQ2W1BV4Bs89LQf+BVxrZo9Wdr4YUbf5yHypZ3pDz57PMPM6qsmTK04ffOADJaa1aQMdO3purF278inX/Lx5HhB33BFefRWOOw423dQDZcuWPlX1/u234YorPEiOHeu5vP3282NXZ+hQD2bpzzlunHe0m90BbwhNSSEj6uY15LukQcAwoCUw3Mwul3QJMMHMxiZNx+8CtgEWAEeY2ZRk32OA3wEGPGZmVf77RYBqPmr7Jb10Kbz7Lvzxj/Dww14cuOOOnpvKTEuXVpzPXrZ0ad1/rrZtoWvXilO3bhXnP/8cbroJLr8chgyBCRNyB+cQmpo6D1ANKQJUKEQm13XaaXDzzYV9wWf2PfFE747pb3/zYsTvvoMVKyq+5nr/2mtw/vmw774eIE84AdZcExYurHz68svcaWnZEvbZB37yEw+ym24afRaGpqmQAIWZldS03XbbWQj5ePZZs+7d/TXXfH3tW5v9V6wwmz/f7KOPzMaPNzv8cDMw69fPrEsXfw9mnTqZ7bmn2QUXmD3yiNncueXHuPLKVc/z7LO+PIRSh5e85RUP4jdaaLTSo/mCv44a5cvrc9/a7N+ypRf39evnLRKfecYbiHz5JYweDf/9L9x5Jxx1FMyf78+K7befDwS54YbecGT6dM9pPfmkH7PQBhrRAjE0FlHEF0IR5NtAZMkSmDjRe9149VVvaj9rVvn6TJHigAGwxRYeyDJT9+7l77t18+BYyLlzqW29YTQOCYUU8UVv5iEUQVU5sPSXd/v2sMsuPoEXAE6f7gHr6qs9aPXq5a0cX3+98jouCdZYozxobbYZDBoEW2/tLSKPO85bRc6YAR06VJzaty9/X1ZWeXDLR6Z5fk33r43aBMcIrMUROagQGqHKGocsX+7N5+fOLX/NTNnzU6d6S8ZCtGgBq60Gy5ZB584+dEq/fh702rb1dautlvt95nXaNBg50jsLfuYZD7SDBnnwbN268nPXNkjUJudYm33rIu1NSbTiC6EJq+2XZfoYmQB3552w7bbezdSSJf6amXLNP/ccvPGGtzbs1w+++caD1rJllb9fvrz6dHXpUrGYMl1UOXeuN80fOhT22MMf6D7zTLjqKs/ZZVpYrlxZ/j57/o03vF5v4EB4/HH47W9hyy1XTYe06rLJk/25t0GDfN+LL/YWl23aVD61bu2vL7xQ+79ZUxEBKoQmrJg5ifT+hTbtX7nSg9RTT8Hxx/sx7rvPg8xaa1XM3WXn+FasqP74paxFC68DXLHCg/DixbD33l53uP76sMEG/tqt26rBsanlviJAhRAqVZsvvLoKboXsb+Z1a5lg9Ze/+PYHH+z7Znr3yASB7Pfp+UmT/Nm1Qw6BMWM8N1VWtur5cpkwAX7/e29BOWaMt778wQ886OY7Pf+8Pz/Xs2d5cWxax47lwSrzumgR/PnPcO+9/qxcQxcv1nWAjOegQgj1orbPYNV2/8zzZhdeWNhza+l9i/HcXGVp/+orszffNHvwQbNrrjE74wyzffc123xzs9VXL38uLjO1a2fWsqXZRhuZ7bqr2X77mR15pNnJJ5uddZbZxRebXX212a23mt13nz9Dd+21/ozdXXeZzZ5t9q9/Ncwzf5WhgOegIgcVQmgUitlQoRjFqmY+mOcnn8CUKXD77V73t/nm3tv/okWrTvl23dWypRc1duqU3/Tpp3DttXDSSV5fWZu6syjiCyE0OY25LqauAlx19X4rVnj9VnbgWrzYA8tDD/l+O+zgLTCrmpYty52WCy+ESy6p0WUAoogvhBCajPoqXqzON9+YzZtnNmWK2W23mXXubHbOObUr3jOLro5CCKHJqG23XOnixEsu8df0WGaVadPGH+6eOhV+9zsft23o0Pz3rwtRxBdCCE1YY27FFwEqhBBCg2nUAUrSXODTYqejiLoD86rdKmSL61Yzcd0KF9esZjLXbT0z65HPDiUXoJo7SRPy/XURysV1q5m4boWLa1YzNblu0UgihBBCSYoAFUIIoSRFgCo9txY7AY1UXLeaietWuLhmNVPwdYs6qBBCCCUpclAhhBBKUgSoEEIIJSkCVAmRNFXSW5ImSYqnlSshabikOZLeTi3rJukpSR8mr12LmcZSU8k1u1jSjOR+myRpUDHTWIokrStpnKR3Jb0j6VfJ8rjfKlHFNSv4fos6qBIiaSpQZmbxEGAVJO0CLAZGmtkWybKhwAIzu0LSeUBXM/ttMdNZSiq5ZhcDi83sqmKmrZRJWgdYx8xel9QRmAgcBBxP3G85VXHNBlPg/RY5qNDomNkLwIKsxQcCI5L3I/B/iJCo5JqFapjZLDN7PXm/CHgP6EXcb5Wq4poVLAJUaTHgSUkTJZ1c7MQ0MmuZ2azk/efAWsVMTCNyuqTJSRFgFFNVQVJfYBvgVeJ+y0vWNYMC77cIUKVlgJltCwwEfpEUy4QCJWPORNl19W4G+gFbA7OAq4ubnNIlqQMwBjjTzL5Kr4v7Lbcc16zg+y0CVAkxsxnJ6xzgQWD74qaoUZmdlH1nysDnFDk9Jc/MZpvZd2a2EriNuN9yktQa/6K9x8z+kSyO+60Kua5ZTe63CFAlQlL7pEIRSe2BvYC3q94rpIwFhiTvhwAPFTEtjULmCzZxMHG/rUKSgDuA98zsmtSquN8qUdk1q8n9Fq34SoSkDfBcE0Ar4F4zu7yISSpZku4DdsO7758NXAT8ExgF9MGHaxlsZtEoIFHJNdsNL24xYCpwSqpeJQCSBgAvAm8BK5PFv8frVOJ+y6GKa3YkBd5vEaBCCCGUpCjiCyGEUJIiQIUQQihJEaBCCCGUpAhQIYQQSlIEqBBCCCUpAlQIIYSSFAEqhBBCSfp/c1+j+R5ubzMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, (ax1, ax2) = plt.subplots(2)\n",
    "# fig.suptitle('Axes values are scaled individually by default')\n",
    "ax1.plot(K, Sum_of_squared_distances, 'bx-')\n",
    "ax1.set_title('Elbow Method For Optimal k ' + name)\n",
    "\n",
    "ax2.plot(K, Silhouette_scores, 'bx-')\n",
    "ax2.set_title('Silhouette Method For Optimal k ' + name)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.savefig(desc + '.svg', dpi=300, format='svg')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sum.pkl', 'rb') as f:\n",
    "    Sum_of_squared_distances = pickle.load(f)\n",
    "    \n",
    "with open('sil.pkl', 'rb') as f:\n",
    "    Silhouette_scores = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_optk('sift')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_optk('surf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_optk('orb')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
